From 21efd44cf2bb614996395ca1c75f40890965d122 Mon Sep 17 00:00:00 2001 From: Eric Pouech Date: Mon, 3 Jul 2023 10:13:00 +0200 Subject: [PATCH] Creation of symbol store generation. CW-Bug-Id: #22341 --- Makefile.in | 24 ++- symstore/Makefile | 4 + symstore/guidelines-deploy.md | 17 ++ symstore/guidelines.md | 74 +++++++ symstore/symstore.c | 383 ++++++++++++++++++++++++++++++++++ 5 files changed, 500 insertions(+), 2 deletions(-) create mode 100644 symstore/Makefile create mode 100644 symstore/guidelines-deploy.md create mode 100644 symstore/guidelines.md create mode 100644 symstore/symstore.c diff --git a/Makefile.in b/Makefile.in index 5b90fcff..452e62ef 100644 --- a/Makefile.in +++ b/Makefile.in @@ -687,6 +687,26 @@ $(OBJ)/.eac-build32: endif +## +## Windows Symbol Store creation +## + +SYMSTORE_DEPENDS = wine + +$(eval $(call rules-source,symstore,$(SRCDIR)/symstore)) +$(eval $(call create-rules-common,symstore,SYMSTORE,64)) + +$(OBJ)/.symstore-build64: + @echo ":: building symstore helper..." >&2 + rsync -arx "$(SYMSTORE_SRC)/" "$(SYMSTORE_OBJ64)/" + $(MAKE) -C "$(SYMSTORE_OBJ64)" SYMSTORE_CFLAGS="-I$(WINE_SRC)/include -I$(WINE_OBJ64)/include" + touch $@ + +.PHONY: symstore-tarball +symstore-tarball: + mkdir -p $(OBJ)/symstore/$(BUILD_NAME) + $(SYMSTORE_OBJ64)/symstore --skip-managed $(DST_BASE) $(OBJ)/symstore/$(BUILD_NAME) + cd $(OBJ)/symstore/$(BUILD_NAME) && zip -r ../$(BUILD_NAME)-symstore.zip . >& /dev/null ## ## Fonts @@ -1130,8 +1150,8 @@ DOCKER_BASE = $(CONTAINER_ENGINE) run --rm -v $(SRC):$(SRC)$(CONTAINER_MOUNT_OPT -w $(OBJ) -e MAKEFLAGS \ $(DOCKER_OPTS) $(STEAMRT_IMAGE) -.PHONY: dist deploy redist -.DEFAULT dist deploy redist: +.PHONY: dist deploy redist symstore-tarball +.DEFAULT dist deploy redist symstore-tarball: if [ "$(ENABLE_CCACHE)" -eq "1" ]; then mkdir -p $(CCACHE_DIR); fi mkdir -p $(CARGO_HOME) $(DOCKER_BASE) $(MAKE) -j$(J) $(filter -j%,$(MAKEFLAGS)) -f $(firstword $(MAKEFILE_LIST)) $(MFLAGS) $(MAKEOVERRIDES) CONTAINER=1 $@ diff --git a/symstore/Makefile b/symstore/Makefile new file mode 100644 index 00000000..2dc42085 --- /dev/null +++ b/symstore/Makefile @@ -0,0 +1,4 @@ +all: symstore + +symstore: symstore.c + $(CC) -o $@ symstore.c $(SYMSTORE_CFLAGS) -D__WINESRC__ -DWINE_UNIX_LIB -Wall -g -Wall -Wno-packed-not-aligned diff --git a/symstore/guidelines-deploy.md b/symstore/guidelines-deploy.md new file mode 100644 index 00000000..d2a74013 --- /dev/null +++ b/symstore/guidelines-deploy.md @@ -0,0 +1,17 @@ +# Building the symbol server + +From a Proton build environment, after building Proton, run +``` +make symstore-tarball +``` +This will create in /symstore a .zip file, named after the +release being built (-symstore.zip). + +# Requirements for symbol server + +This requires mapping a directory (and all its sub-hierarchy) at +a given URI. + +There are two strong requirements: +- all files in must be served as binary, +- the mapping of must be case insensitive. diff --git a/symstore/guidelines.md b/symstore/guidelines.md new file mode 100644 index 00000000..58c57b7b --- /dev/null +++ b/symstore/guidelines.md @@ -0,0 +1,74 @@ +This small note to describe how a windows developer can inject +minidump files generated from a crash of a game running under Proton +Steam (desktop or Deck). + +Note: this doesn't apply if the application is running managed code as +it's required more compabilities between .NET and Mono. So, from now +on, we only consider native (non managed) code. + +These minidumps are fully reloadable under Microsoft tools (WinDbg or +Visual Studio) when the binaries of the Proton system files are made +available to the debugging tool. +It's mandatory for reloading a minidump created with a given Proton +version that the very same Proton system files are made avaiable to +the machine running the debugging tool. + +If the right version is not installed, the debugging tool will likely +emit warnings (like "couldn't match module", or "invalid time +stamp"). This means the debugging tool will report likely bogus +information, and your setup must be fixed before going any further. + +# Installing on dev machine + +Each new released proton version (starting at Proton 8.0-4) will also be +available as a zip package to be installed inside symbol store. + +The .zip file is available as a part of GitHub release and is named +-symstore.zip. + +You must create a top directory (referred as below) and +extract the zip file with as the parent directory. + +Each time a new Proton's version is released, this extraction scheme +has to be done in order to have proper access to minidumps generated +from this Proton version. + +# Setting up the debugging tools + +## WinDbg + +In the WinDbg "File > Settings" menu, in the "Symbols path" pane, add +``` +srv* +``` +Of course, one can add also the paths to the application being +debugged. + +## Visual Studio + +In Visual Studio: +- Open the minidump from the "File > Open > File" menu +- Open the "Set Symbol paths" item, and add a (.pdb) store pointing to + +- Depending on your project settings, you could also have another + store pointing to your own application +- Then start debugging by using the "Debug with..." appropriate to + your project. + +## Other debugging tool + +If the tool supports symbol stores, then there's should be a similar +configuration as described above for MS tools. + +[[ Note: the settings above should work also if we're able to generate +.pdb files along side the images ]] + +# Public store + +A Proton symbol store is available at . + +Then no local installation is required, only setting the +``` +srv* +``` +in the debugging tools would be required. diff --git a/symstore/symstore.c b/symstore/symstore.c new file mode 100644 index 00000000..98dae86e --- /dev/null +++ b/symstore/symstore.c @@ -0,0 +1,383 @@ +/* + * Eric Pouech for CodeWeavers + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA + */ + +#include "config.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +#define _XOPEN_SOURCE 700 +#include +#include +#include + + +#include "windef.h" +#include "winbase.h" +#include "winnt.h" + +static BOOL verbose; +static BOOL skip_managed_code; +static unsigned skipped_files; +static unsigned inserted_files; +static unsigned conflict_files; +static enum {MAP_ASIS, MAP_LOWERCASE, MAP_UPPERCASE} case_sensitivity; + +static void usage(const char* error, ...) +{ + if (error) + { + va_list valist; + + va_start( valist, error ); + vfprintf( stderr, error, valist ); + va_end( valist ); + } + fprintf(stderr, + "symstore [options] \n" + " the top of a Proton dist tree\n" + " the top of a symstore-like tree generated from \n"); + fprintf(stderr, + "options:\n" + " -v prints verbose information\n" + " --lower-case write every filename in in lower case\n" + " --upper-case write every filename in in upper case\n" + " (by default, case of filename is preserved\n"); + + exit(1); +} + +static BOOL ensure_path_exists(const char* in) +{ + char path[MAX_PATH]; + char* p; + + p = strcpy(path, in); + if (*p == '/') p++; + for (;;) + { + p = strchr(p, '/'); + if (!p) break; + *p = '\0'; + if (mkdir(path, 0777) < 0) + { + struct stat st; + if (errno != EEXIST || stat(path, &st) < 0 || (st.st_mode & S_IFMT) != S_IFDIR) + { + printf("failed to create %s from %s\n", path, in); + return FALSE; + } + } + *p++ = '/'; + } + return TRUE; +} + +static BOOL copy_file(const char* from, const char* to) +{ + struct stat st; + FILE* f1; + FILE* f2; + BOOL ret = FALSE; + + if (stat(to, &st) == 0) return FALSE; + f1 = fopen(from, "rb"); + f2 = fopen(to, "wb"); + if (f1 && f2) + { + char buffer[512]; + size_t len1, len2; + + while ((len1 = fread(buffer, 1, ARRAYSIZE(buffer), f1))) + { + len2 = fwrite(buffer, 1, len1, f2); + if (len1 != len2) break; + } + ret = feof(f1); + } + if (f1) fclose(f1); + if (f2) fclose(f2); + return ret; +} + +static BOOL files_same_content(const char* file1, const char* file2) +{ + FILE* f1 = fopen(file1, "rb"); + FILE* f2 = fopen(file2, "rb"); + BOOL ret = FALSE; + + if (f1 && f2) + { + char buffer1[512], buffer2[512]; + size_t len1, len2; + + for (;;) + { + len1 = fread(buffer1, 1, ARRAYSIZE(buffer1), f1); + len2 = fread(buffer2, 1, ARRAYSIZE(buffer2), f2); + if (len1 != len2) break; + if (len1 && memcmp(buffer1, buffer2, len1)) break; + if ((ret = (feof(f1) && feof(f2)))) break; + } + } + if (f1) fclose(f1); + if (f2) fclose(f2); + return ret; +} + +static BOOL validate_proton_dist(const char* src) +{ + char buffer[MAX_PATH]; + struct stat st; + BOOL allgood = TRUE; + + snprintf(buffer, ARRAYSIZE(buffer), "%s/%s", src, "version"); + allgood = stat(buffer, &st) == 0 && (st.st_mode & S_IFMT) == S_IFREG; + + snprintf(buffer, ARRAYSIZE(buffer), "%s/%s", src, "dist"); + allgood = allgood && stat(buffer, &st) == 0 && (st.st_mode & S_IFMT) == S_IFDIR; + + return allgood; +} + +static const IMAGE_DATA_DIRECTORY* get_directory(const IMAGE_NT_HEADERS64* nthdr, unsigned entry) +{ + if (nthdr->OptionalHeader.Magic == IMAGE_NT_OPTIONAL_HDR64_MAGIC) + return entry < nthdr->OptionalHeader.NumberOfRvaAndSizes ? + &nthdr->OptionalHeader.DataDirectory[entry] : NULL; + if (nthdr->OptionalHeader.Magic == IMAGE_NT_OPTIONAL_HDR32_MAGIC) + { + const IMAGE_NT_HEADERS32* nthdr32 = (const IMAGE_NT_HEADERS32*)nthdr; + return entry < nthdr32->OptionalHeader.NumberOfRvaAndSizes ? + &nthdr32->OptionalHeader.DataDirectory[entry] : NULL; + } + return NULL; +} + +static BOOL get_pe_srvinfo(const char* from, unsigned* timestamp, unsigned* size) +{ + static char header_buffer[1024]; + int fd; + ssize_t header_read; + const IMAGE_DOS_HEADER* dos; + const IMAGE_NT_HEADERS64* nthdr; + BOOL ret = FALSE; + + /* HACK!!! */ + /* find a better way to drop out debug only PE images */ + if (strlen(from) > 6 && !strcasecmp(from + strlen(from) - 6, ".debug")) return FALSE; + + if ((fd = open(from, O_RDONLY/* | O_BINARY */)) == -1) return FALSE; + header_read = read(fd, header_buffer, sizeof(header_buffer)); + if (header_read >= 0) + { +#define X(t, p) (((p) + sizeof(t) < header_read) ? (const t*)(header_buffer + p) : NULL) + + if ((dos = X(IMAGE_DOS_HEADER, 0)) != NULL && + dos->e_magic == IMAGE_DOS_SIGNATURE && + (nthdr = X(IMAGE_NT_HEADERS64, dos->e_lfanew)) != NULL && + nthdr->Signature == IMAGE_NT_SIGNATURE) + { + const IMAGE_DATA_DIRECTORY* dir = get_directory(nthdr, IMAGE_DIRECTORY_ENTRY_COM_DESCRIPTOR); + *timestamp = nthdr->FileHeader.TimeDateStamp; + + ret = !(skip_managed_code && dir && dir->Size); + + if (nthdr->OptionalHeader.Magic == IMAGE_NT_OPTIONAL_HDR64_MAGIC) + { + *size = nthdr->OptionalHeader.SizeOfImage; + } + else if (nthdr->OptionalHeader.Magic == IMAGE_NT_OPTIONAL_HDR32_MAGIC) + { + const IMAGE_NT_HEADERS32* nthdr32 = (const IMAGE_NT_HEADERS32*)nthdr; + *size = nthdr32->OptionalHeader.SizeOfImage; + } + else ret = FALSE; + } +#undef X + } + close(fd); + return ret; +} + +static BOOL get_discriminant(const char* from, char* buffer, unsigned len) +{ + unsigned timestamp, size; + + if (get_pe_srvinfo(from, ×tamp, &size)) + { + /* PE image */ + if (timestamp && size) + return snprintf(buffer, len, "%X%x", timestamp, size) < len; + if (!timestamp && size) + fprintf(stderr, "Image %s has 0-timestamp\n", from); + } +#if 0 + static GUID null_guid; + /* PDB DS */ + if (!IsEqualGUID(&info.guid, &null_guid) && info.age) + return snprintf(buffer, len, "%08x%04x%04x%02x%02x%02x%02x%02x%02x%02x%02x%lx", + info.guid.Data1, info.guid.Data2, info.guid.Data3, + info.guid.Data4[0], info.guid.Data4[1], + info.guid.Data4[2], info.guid.Data4[3], + info.guid.Data4[4], info.guid.Data4[5], + info.guid.Data4[6], info.guid.Data4[7], + info.age) < len; + + /* PDS JG and DBG are likely not needed */ + + } +#endif + if (verbose) + fprintf(stderr, "Not a MS image or debug file %s\n", from); + skipped_files++; + + return FALSE; +} + +static void insert(const char* from, const char* dst) +{ + char discrim[64]; + + if (get_discriminant(from, discrim, ARRAYSIZE(discrim))) + { + char out[MAX_PATH]; + const char* last = strrchr(from, '/'); + assert(last); + + snprintf(out, ARRAYSIZE(out), "%s/%s/%s/%s", dst, last + 1, discrim, last + 1); + + if (verbose) + printf("Copying %s into %s\n", from, out); + if (!ensure_path_exists(out)) + { + fprintf(stderr, "Couldn't create directories for %s\n", out); + } + else if (copy_file(from, out) || files_same_content(from, out)) + inserted_files++; + else + { + fprintf(stderr, "Conflict for file %s (%s)\n", out, from); + conflict_files++; + } + } +} + +static void recurse(char* buffer, const char* dst) +{ + DIR* dp; + struct dirent* ep; + size_t at; + struct stat st; + + if (verbose > 1) + fprintf(stderr, "Recursing into %s\n", buffer); + + at = strlen(buffer); + + if ((dp = opendir(buffer)) == NULL) + { + fprintf(stderr, "'%s' doesn't exist\n", buffer); + usage(NULL); + } + while ((ep = readdir(dp)) != NULL) + { + if (!strcmp(ep->d_name, ".") || !strcmp(ep->d_name, "..")) continue; + + if (at + 1 + strlen(ep->d_name) < MAX_PATH) + { + char* ptr; + + buffer[at] = '/'; + strcpy(buffer + at + 1, ep->d_name); + switch (case_sensitivity) + { + case MAP_ASIS: + break; + case MAP_LOWERCASE: + for (ptr = buffer + at; *ptr; ptr++) + *ptr = tolower(*ptr); + break; + case MAP_UPPERCASE: + for (ptr = buffer + at; *ptr; ptr++) + *ptr = toupper(*ptr); + break; + } + if (stat(buffer, &st) == 0 && (st.st_mode & S_IFMT) == S_IFDIR) + recurse(buffer, dst); + else + insert(buffer, dst); + } + else + { + fprintf(stderr, "Too long filename at %s %s\n", buffer, ep->d_name); + } + } + closedir (dp); + buffer[at] = '\0'; +} + +int main(int argc, char* argv[]) +{ + char src_buffer[MAX_PATH]; + const char* src = NULL; + const char* dst = NULL; + int i; + + for (i = 1; i < argc; i++) + { + if (argv[i][0] == '-') + { + if (!strcmp(argv[i], "-v")) verbose++; + else if (!strcmp(argv[i], "--skip-managed")) skip_managed_code = TRUE; + else if (!strcmp(argv[i], "--lower-case")) case_sensitivity = MAP_LOWERCASE; + else if (!strcmp(argv[i], "--upper-case")) case_sensitivity = MAP_UPPERCASE; + else usage("Unknown option %s\n", argv[i]); + } + else + { + if (!src) src = argv[i]; + else if (!dst) dst = argv[i]; + else usage("Incorrect argument %s\n", argv[i]); + } + } + if (!dst) usage("Missing argument\n"); + if (!validate_proton_dist(src)) usage("%s isn't a Proton directory\n", src); + + strcpy(src_buffer, src); + if (src_buffer[strlen(src_buffer) - 1] == '/') src_buffer[strlen(src_buffer) - 1] = '\0'; + + recurse(src_buffer, dst); + + printf("Status for inserting %s into %s\n", src, dst); + printf(" %4u files copied\n", inserted_files); + printf(" %4u files skipped\n", skipped_files); + if (conflict_files) + printf(" %4u files in conflict\n", conflict_files); + + return 0; +}