yuzu-emu/breakpad
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Add optional new symbol upload API to sym_upload.

Browse source 
Change-Id: I6a49e9f4a699fa6f5f8e9f0fc86afb4cb342a442
Reviewed-on: https://chromium-review.googlesource.com/c/breakpad/breakpad/+/1422400
Reviewed-by: Mark Mentovai <mark@chromium.org>
Reviewed-by: Ivan Penkov <ivanpe@chromium.org>
Reviewed-by: Mike Frysinger <vapier@chromium.org>
This commit is contained in:
Nelson Billing 2020-02-19 15:32:08 -08:00
parent 216cea7bca
commit bbad9f255d
11 changed files with 880 additions and 99 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
Makefile.am
View file

@ -615,6 +615,10 @@ src_tools_linux_symupload_minidump_upload_LDADD = -ldl
src_tools_linux_symupload_sym_upload_SOURCES = \
src/common/linux/http_upload.cc \
src/common/linux/http_upload.h \
src/common/linux/libcurl_wrapper.cc \
src/common/linux/libcurl_wrapper.h \
src/common/linux/symbol_collector_client.cc \
src/common/linux/symbol_collector_client.h \
src/common/linux/symbol_upload.cc \
src/common/linux/symbol_upload.h \
src/tools/linux/symupload/sym_upload.cc

14
Makefile.in
View file

@ -1495,10 +1495,16 @@ src_tools_linux_symupload_minidump_upload_OBJECTS = \
src_tools_linux_symupload_minidump_upload_DEPENDENCIES =
am__src_tools_linux_symupload_sym_upload_SOURCES_DIST = \
src/common/linux/http_upload.cc src/common/linux/http_upload.h \
src/common/linux/libcurl_wrapper.cc \
src/common/linux/libcurl_wrapper.h \
src/common/linux/symbol_collector_client.cc \
src/common/linux/symbol_collector_client.h \
src/common/linux/symbol_upload.cc \
src/common/linux/symbol_upload.h \
src/tools/linux/symupload/sym_upload.cc
@DISABLE_TOOLS_FALSE@@LINUX_HOST_TRUE@am_src_tools_linux_symupload_sym_upload_OBJECTS = src/common/linux/http_upload.$(OBJEXT) \
@DISABLE_TOOLS_FALSE@@LINUX_HOST_TRUE@ src/common/linux/libcurl_wrapper.$(OBJEXT) \
@DISABLE_TOOLS_FALSE@@LINUX_HOST_TRUE@ src/common/linux/symbol_collector_client.$(OBJEXT) \
@DISABLE_TOOLS_FALSE@@LINUX_HOST_TRUE@ src/common/linux/symbol_upload.$(OBJEXT) \
@DISABLE_TOOLS_FALSE@@LINUX_HOST_TRUE@ src/tools/linux/symupload/sym_upload.$(OBJEXT)
src_tools_linux_symupload_sym_upload_OBJECTS = \
@ -2477,6 +2483,10 @@ TESTS = $(check_PROGRAMS) $(check_SCRIPTS)
@DISABLE_TOOLS_FALSE@@LINUX_HOST_TRUE@src_tools_linux_symupload_sym_upload_SOURCES = \
@DISABLE_TOOLS_FALSE@@LINUX_HOST_TRUE@ src/common/linux/http_upload.cc \
@DISABLE_TOOLS_FALSE@@LINUX_HOST_TRUE@ src/common/linux/http_upload.h \
@DISABLE_TOOLS_FALSE@@LINUX_HOST_TRUE@ src/common/linux/libcurl_wrapper.cc \
@DISABLE_TOOLS_FALSE@@LINUX_HOST_TRUE@ src/common/linux/libcurl_wrapper.h \
@DISABLE_TOOLS_FALSE@@LINUX_HOST_TRUE@ src/common/linux/symbol_collector_client.cc \
@DISABLE_TOOLS_FALSE@@LINUX_HOST_TRUE@ src/common/linux/symbol_collector_client.h \
@DISABLE_TOOLS_FALSE@@LINUX_HOST_TRUE@ src/common/linux/symbol_upload.cc \
@DISABLE_TOOLS_FALSE@@LINUX_HOST_TRUE@ src/common/linux/symbol_upload.h \
@DISABLE_TOOLS_FALSE@@LINUX_HOST_TRUE@ src/tools/linux/symupload/sym_upload.cc
@ -4726,6 +4736,9 @@ src/tools/linux/symupload/minidump_upload.$(OBJEXT): \
src/tools/linux/symupload/minidump_upload$(EXEEXT): $(src_tools_linux_symupload_minidump_upload_OBJECTS) $(src_tools_linux_symupload_minidump_upload_DEPENDENCIES) $(EXTRA_src_tools_linux_symupload_minidump_upload_DEPENDENCIES) src/tools/linux/symupload/$(am__dirstamp)
@rm -f src/tools/linux/symupload/minidump_upload$(EXEEXT)
$(AM_V_CXXLD)$(CXXLINK) $(src_tools_linux_symupload_minidump_upload_OBJECTS) $(src_tools_linux_symupload_minidump_upload_LDADD) $(LIBS)
src/common/linux/symbol_collector_client.$(OBJEXT): \
src/common/linux/$(am__dirstamp) \
src/common/linux/$(DEPDIR)/$(am__dirstamp)
src/common/linux/symbol_upload.$(OBJEXT): \
src/common/linux/$(am__dirstamp) \
src/common/linux/$(DEPDIR)/$(am__dirstamp)
@ -4996,6 +5009,7 @@ distclean-compile:
@AMDEP_TRUE@@am__include@ @am__quote@src/common/linux/$(DEPDIR)/src_tools_linux_dump_syms_dump_syms-linux_libc_support.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@src/common/linux/$(DEPDIR)/src_tools_linux_dump_syms_dump_syms-memory_mapped_file.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@src/common/linux/$(DEPDIR)/src_tools_linux_dump_syms_dump_syms-safe_readlink.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@src/common/linux/$(DEPDIR)/symbol_collector_client.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@src/common/linux/$(DEPDIR)/symbol_upload.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@src/common/linux/tests/$(DEPDIR)/src_client_linux_linux_client_unittest_shlib-crash_generator.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@src/common/linux/tests/$(DEPDIR)/src_common_dumper_unittest-crash_generator.Po@am__quote@

214
docs/sym_upload_v2_protocol.md Normal file
View file

@ -0,0 +1,214 @@
# Introduction
The `sym_upload` tool is able to operate in `sym-upload-v2` protocol mode, in
addition to the legacy protocol (which will be referred to as `sym-upload-v1`
for the rest of this document). For now `sym-upload-v2` is HTTP/REST-based but
it could be extended to operate over gRPC instead, in the future.
# Table of Contents
* [Why](#why)
* [How](#how)
* [Uploading](#uploading)
* [Uploading with `sym_upload`](#uploading-with-sym_upload)
* [Uploading with curl](#uploading-with-curl)
* [Serving the `sym-upload-v2` protocol](#serving-the-sym-upload-v2-protocol)
* [Authenticate using `key`](#authenticate-using-key)
* [Symbol `checkStatus`](#symbol-checkstatus)
* [Upload `create`](#upload-create)
* [Uploading the symbol file](#uploading-the-symbol-file)
* [Upload complete](#upload-complete)
# Why
Using `sym_upload` in `sym-upload-v2` protocol mode has the following features
beyond `sym-upload-v1`:
* Authentication via `key` (arbitrary secret).
* Symbol identifier (product of `debug_file` and `debug_id`, as recorded in
output from `dump_syms`) can be checked against existing symbol information on
server. If it's present, then the upload is skipped entirely.
# How
## Uploading
### Uploading with `sym_upload`
Uploading in `sym-upload-v2` protocol mode is easy. Invoke `sym_upload` like
```
$ ./sym_upload -p sym-upload-v2 [-k <API-key>] <symbol-file> <API-URL>
```
Where `symbol-file` is a symbol file created by `dump_syms`, `API-URL` is the
URL of your `sym-upload-v2` API service (see next section for details), and
`API-key` is a secret known to your uploader and server.
For more options see `sym_upload --help`.
### Uploading with curl
As an example, if:
* Your API's URL was "https://sym-upload-api".
* Your service has assigned you `key` "myfancysecret123".
* You wanted to upload the symbol file at "path/to/file_name", with
`debug_file` being "file_name" and `debug_id` being
"123123123123123123123123123". Normally you would read these values from
"path/to/file_name", which in turn was generated by `dump_syms`.
Then you might run:
```
$ curl https://sym-upload-api/symbols/file_name/123123123123123123123123123:checkStatus?key=myfancysecret123
```
And, upon seeing that this `debug_file`/`debug_id` combo is missing from symbol
storage then you could run:
```
$ curl --request POST https://sym-upload-api/uploads:create?key=myfancysecret123
```
Which returns `upload_url` "https://upload-server/42?creds=shhhhh" and
`upload_key` "42". Next you upload the file directly like:
```
$ curl -T path/to/file_name "https://upload-server/42?creds=shhhhh"
```
Once the HTTP PUT is complete, run:
```
$ curl --header "Content-Type: application/json" \
--request POST \
--data '{symbol_id:{"debugFile":"file_name",'\
'"debugId":"123123123123123123123123123"}}' \
https://sym-upload-api/uploads/42:complete?key=myfancysecret123
```
### Serving the `sym-upload-v2` Protocol
The protocol is currently defined only in HTTP/REST. There are three necessary
REST operations to implement in your service:
* `/symbols/<debug_file>/<debug_id>:checkStatus?key=<key>`
* `/uploads:create?key=<key>`
* `/uploads/<upload_key>:complete?key=<key>`
#### Authenticate Using `key`
The query string arg `key` contains some secret that both the uploader and
server understand. It is up to the service implementer to decide on what
constitutes a valid `key`, how the uploader acquires one, and how to handle
requests made with invalid ones.
#### Symbol `checkStatus`
```
/symbols/<debug_file>/<debug_id>:checkStatus?key=<key>
```
This operation expects an empty (or no) JSON payload in the request.
This operation should return the status of the symbol file uniquely identified
by the given `debug_file` and `debug_id`. JSON schema:
```
{
"type": object",
"properties": {
"status": {
"type": "string",
"enum": ["STATUS_UNSPECIFIED", "MISING", "FOUND"],
"required": true
}
}
}
```
Where `MISSING` denotes that the symbol file does not exist on the server and
`FOUND` denotes that the symbol file exists on the server.
#### Upload `create`
```
/uploads:create?key=<key>
```
This operation expects an empty (or no) JSON payload in the request.
This operation should return a URL that uploader can HTTP PUT their symbol file
to, along with an "upload key" that can be used to notify the service once the
file upload is completed. JSON schema:
```
{
"type": "object",
"properties": {
"upload_url": {
"type: "string",
"required": true
},
"upload_key": {
"type": "string",
"required": true
}
}
}
```
Since this REST API operation can be authenticated via the `key` query string
arg, the service can return a URL that encodes permission delegation to the
upload endpoint resource and thereby constrain the ability to upload to those
with valid `key`s.
#### Uploading the Symbol File
Note that the actual symbol upload step is _not_ part of the REST API. The
upload URL obtained in the above operation is meant to be used as the endpoint
for a normal HTTP PUT request for the contents of the symbol file. Once that
HTTP PUT request is completed use the upload `complete` operation.
#### Upload `complete`
```
/uploads/<upload_key>:complete?key=<key>
```
This operation expects a JSON payload in the HTTP request body with the
following schema:
```
{
"type": "object",
"properties": {
"symbol_id": {
"type": "object",
"properties": {
"debug_file": {
"type": "string",
"required": true
},
"debug_id": {
"type": "string",
"required": true
}
}
}
}
}
```
This operation should cause the symbol storage back-end (however implemented)
to consume the symbol file identified by `upload_key`. It is up to the service
implementation to decide how uploads are assigned `upload_key`s and how to
retrieve a completed upload by its `upload_key`. If the symbol file cannot be
found, is malformed, or the operation cannot be completed for any other reason
then an HTTP error will be returned. JSON schema of non-error responses:
```
{
"type": "object",
"properties": {
"result": {
"type": string,
"enum": ["RESULT_UNSPECIFIED", "OK", "DUPLICATE_DATA"],
"required": true
}
}
}
```
Where `OK` denotes that the symbol storage was updated with the new symbol file
and `DUPLICATE_DATA` denotes that the symbol file data was identical to data
already in symbol storage and therefore nothing changed.

2
src/common/common.gyp
View file

@ -118,6 +118,8 @@
'linux/memory_mapped_file.h',
'linux/safe_readlink.cc',
'linux/safe_readlink.h',
'linux/symbol_collector_client.cc',
'linux/symbol_collector_client.h',
'linux/synth_elf.cc',
'linux/synth_elf.h',
'long_string_dictionary.cc',

227
src/common/linux/libcurl_wrapper.cc
View file

@ -38,32 +38,24 @@
namespace google_breakpad {
LibcurlWrapper::LibcurlWrapper()
: init_ok_(false),
formpost_(NULL),
lastptr_(NULL),
headerlist_(NULL) {
curl_lib_ = dlopen("libcurl.so", RTLD_NOW);
if (!curl_lib_) {
curl_lib_ = dlopen("libcurl.so.4", RTLD_NOW);
}
if (!curl_lib_) {
curl_lib_ = dlopen("libcurl.so.3", RTLD_NOW);
}
if (!curl_lib_) {
std::cout << "Could not find libcurl via dlopen";
return;
}
std::cout << "LibcurlWrapper init succeeded";
init_ok_ = true;
return;
}
curl_lib_(nullptr),
last_curl_error_(""),
curl_(nullptr),
formpost_(nullptr),
lastptr_(nullptr),
headerlist_(nullptr) {}
LibcurlWrapper::~LibcurlWrapper() {}
LibcurlWrapper::~LibcurlWrapper() {
if (init_ok_) {
(*easy_cleanup_)(curl_);
dlclose(curl_lib_);
}
}
bool LibcurlWrapper::SetProxy(const string& proxy_host,
const string& proxy_userpwd) {
if (!init_ok_) {
return false;
}
if (!CheckInit()) return false;
// Set proxy information if necessary.
if (!proxy_host.empty()) {
(*easy_setopt_)(curl_, CURLOPT_PROXY, proxy_host.c_str());
@ -83,9 +75,8 @@ bool LibcurlWrapper::SetProxy(const string& proxy_host,
bool LibcurlWrapper::AddFile(const string& upload_file_path,
const string& basename) {
if (!init_ok_) {
return false;
}
if (!CheckInit()) return false;
std::cout << "Adding " << upload_file_path << " to form upload.";
// Add form file.
(*formadd_)(&formpost_, &lastptr_,
@ -110,10 +101,11 @@ static size_t WriteCallback(void *ptr, size_t size,
bool LibcurlWrapper::SendRequest(const string& url,
const std::map<string, string>& parameters,
int* http_status_code,
long* http_status_code,
string* http_header_data,
string* http_response_data) {
(*easy_setopt_)(curl_, CURLOPT_URL, url.c_str());
if (!CheckInit()) return false;
std::map<string, string>::const_iterator iter = parameters.begin();
for (; iter != parameters.end(); ++iter)
(*formadd_)(&formpost_, &lastptr_,
@ -122,55 +114,79 @@ bool LibcurlWrapper::SendRequest(const string& url,
CURLFORM_END);
(*easy_setopt_)(curl_, CURLOPT_HTTPPOST, formpost_);
if (http_response_data != NULL) {
http_response_data->clear();
(*easy_setopt_)(curl_, CURLOPT_WRITEFUNCTION, WriteCallback);
(*easy_setopt_)(curl_, CURLOPT_WRITEDATA,
reinterpret_cast<void *>(http_response_data));
}
if (http_header_data != NULL) {
http_header_data->clear();
(*easy_setopt_)(curl_, CURLOPT_HEADERFUNCTION, WriteCallback);
(*easy_setopt_)(curl_, CURLOPT_HEADERDATA,
reinterpret_cast<void *>(http_header_data));
return SendRequestInner(url, http_status_code, http_header_data,
http_response_data);
}
bool LibcurlWrapper::SendGetRequest(const string& url,
long* http_status_code,
string* http_header_data,
string* http_response_data) {
if (!CheckInit()) return false;
(*easy_setopt_)(curl_, CURLOPT_HTTPGET, 1L);
return SendRequestInner(url, http_status_code, http_header_data,
http_response_data);
}
bool LibcurlWrapper::SendPutRequest(const string& url,
const string& path,
long* http_status_code,
string* http_header_data,
string* http_response_data) {
if (!CheckInit()) return false;
FILE* file = fopen(path.c_str(), "rb");
(*easy_setopt_)(curl_, CURLOPT_UPLOAD, 1L);
(*easy_setopt_)(curl_, CURLOPT_PUT, 1L);
(*easy_setopt_)(curl_, CURLOPT_READDATA, file);
bool success = SendRequestInner(url, http_status_code, http_header_data,
http_response_data);
fclose(file);
return success;
}
bool LibcurlWrapper::SendSimplePostRequest(const string& url,
const string& body,
const string& content_type,
long* http_status_code,
string* http_header_data,
string* http_response_data) {
if (!CheckInit()) return false;
(*easy_setopt_)(curl_, CURLOPT_POSTFIELDSIZE, body.size());
(*easy_setopt_)(curl_, CURLOPT_COPYPOSTFIELDS, body.c_str());
if (!content_type.empty()) {
string content_type_header = "Content-Type: " + content_type;
headerlist_ = (*slist_append_)(
headerlist_,
content_type_header.c_str());
}
CURLcode err_code = CURLE_OK;
err_code = (*easy_perform_)(curl_);
easy_strerror_ = reinterpret_cast<const char* (*)(CURLcode)>
(dlsym(curl_lib_, "curl_easy_strerror"));
if (http_status_code != NULL) {
(*easy_getinfo_)(curl_, CURLINFO_RESPONSE_CODE, http_status_code);
}
#ifndef NDEBUG
if (err_code != CURLE_OK)
fprintf(stderr, "Failed to send http request to %s, error: %s\n",
url.c_str(),
(*easy_strerror_)(err_code));
#endif
if (headerlist_ != NULL) {
(*slist_free_all_)(headerlist_);
}
(*easy_cleanup_)(curl_);
if (formpost_ != NULL) {
(*formfree_)(formpost_);
}
return err_code == CURLE_OK;
return SendRequestInner(url, http_status_code, http_header_data,
http_response_data);
}
bool LibcurlWrapper::Init() {
if (!init_ok_) {
std::cout << "Init_OK was not true in LibcurlWrapper::Init(), check earlier log messages";
curl_lib_ = dlopen("libcurl.so", RTLD_NOW);
if (!curl_lib_) {
curl_lib_ = dlopen("libcurl.so.4", RTLD_NOW);
}
if (!curl_lib_) {
curl_lib_ = dlopen("libcurl.so.3", RTLD_NOW);
}
if (!curl_lib_) {
std::cout << "Could not find libcurl via dlopen";
return false;
}
if (!SetFunctionPointers()) {
std::cout << "Could not find function pointers";
init_ok_ = false;
return false;
}
@ -184,11 +200,7 @@ bool LibcurlWrapper::Init() {
return false;
}
// Disable 100-continue header.
char buf[] = "Expect:";
headerlist_ = (*slist_append_)(headerlist_, buf);
(*easy_setopt_)(curl_, CURLOPT_HTTPHEADER, headerlist_);
init_ok_ = true;
return true;
}
@ -228,6 +240,10 @@ bool LibcurlWrapper::SetFunctionPointers() {
"curl_easy_getinfo",
CURLcode(*)(CURL *, CURLINFO info, ...));
SET_AND_CHECK_FUNCTION_POINTER(easy_reset_,
"curl_easy_reset",
void(*)(CURL*));
SET_AND_CHECK_FUNCTION_POINTER(slist_free_all_,
"curl_slist_free_all",
void(*)(curl_slist*));
@ -238,4 +254,73 @@ bool LibcurlWrapper::SetFunctionPointers() {
return true;
}
bool LibcurlWrapper::SendRequestInner(const string& url,
long* http_status_code,
string* http_header_data,
string* http_response_data) {
string url_copy(url);
(*easy_setopt_)(curl_, CURLOPT_URL, url_copy.c_str());
// Disable 100-continue header.
char buf[] = "Expect:";
headerlist_ = (*slist_append_)(headerlist_, buf);
(*easy_setopt_)(curl_, CURLOPT_HTTPHEADER, headerlist_);
if (http_response_data != nullptr) {
http_response_data->clear();
(*easy_setopt_)(curl_, CURLOPT_WRITEFUNCTION, WriteCallback);
(*easy_setopt_)(curl_, CURLOPT_WRITEDATA,
reinterpret_cast<void*>(http_response_data));
}
if (http_header_data != nullptr) {
http_header_data->clear();
(*easy_setopt_)(curl_, CURLOPT_HEADERFUNCTION, WriteCallback);
(*easy_setopt_)(curl_, CURLOPT_HEADERDATA,
reinterpret_cast<void*>(http_header_data));
}
CURLcode err_code = CURLE_OK;
err_code = (*easy_perform_)(curl_);
easy_strerror_ = reinterpret_cast<const char* (*)(CURLcode)>
(dlsym(curl_lib_, "curl_easy_strerror"));
if (http_status_code != nullptr) {
(*easy_getinfo_)(curl_, CURLINFO_RESPONSE_CODE, http_status_code);
}
#ifndef NDEBUG
if (err_code != CURLE_OK)
fprintf(stderr, "Failed to send http request to %s, error: %s\n",
url.c_str(),
(*easy_strerror_)(err_code));
#endif
Reset();
return err_code == CURLE_OK;
}
void LibcurlWrapper::Reset() {
if (headerlist_ != nullptr) {
(*slist_free_all_)(headerlist_);
headerlist_ = nullptr;
}
if (formpost_ != nullptr) {
(*formfree_)(formpost_);
formpost_ = nullptr;
}
(*easy_reset_)(curl_);
}
bool LibcurlWrapper::CheckInit() {
if (!init_ok_) {
std::cout << "LibcurlWrapper: You must call Init(), and have it return "
"'true' before invoking any other methods.\n";
return false;
}
return true;
}
} // namespace google_breakpad

28
src/common/linux/libcurl_wrapper.h
View file

@ -51,14 +51,39 @@ class LibcurlWrapper {
const string& basename);
virtual bool SendRequest(const string& url,
const std::map<string, string>& parameters,
int* http_status_code,
long* http_status_code,
string* http_header_data,
string* http_response_data);
bool SendGetRequest(const string& url,
long* http_status_code,
string* http_header_data,
string* http_response_data);
bool SendPutRequest(const string& url,
const string& path,
long* http_status_code,
string* http_header_data,
string* http_response_data);
bool SendSimplePostRequest(const string& url,
const string& body,
const string& content_type,
long* http_status_code,
string* http_header_data,
string* http_response_data);
private:
// This function initializes class state corresponding to function
// pointers into the CURL library.
bool SetFunctionPointers();
bool SendRequestInner(const string& url,
long* http_status_code,
string* http_header_data,
string* http_response_data);
void Reset();
bool CheckInit();
bool init_ok_; // Whether init succeeded
void* curl_lib_; // Pointer to result of dlopen() on
// curl library
@ -85,6 +110,7 @@ class LibcurlWrapper {
const char* (*easy_strerror_)(CURLcode);
void (*easy_cleanup_)(CURL *);
CURLcode (*easy_getinfo_)(CURL *, CURLINFO info, ...);
void (*easy_reset_)(CURL*);
void (*formfree_)(struct curl_httppost *);
};

193
src/common/linux/symbol_collector_client.cc Normal file
View file

@ -0,0 +1,193 @@
// Copyright (c) 2019 Google Inc.
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
// * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include "common/linux/symbol_collector_client.h"
#include <stdio.h>
#include <iostream>
#include <regex>
#include "common/linux/libcurl_wrapper.h"
namespace google_breakpad {
namespace sym_upload {
// static
bool SymbolCollectorClient::CreateUploadUrl(
LibcurlWrapper* libcurl_wrapper,
const string& api_url,
const string& api_key,
UploadUrlResponse* uploadUrlResponse) {
string header, response;
long response_code;
string url = api_url + "/v1/uploads:create";
if (!api_key.empty()) {
url += "?key=" + api_key;
}
if (!libcurl_wrapper->SendSimplePostRequest(url,
/*body=*/"",
/*content_type=*/"",
&response_code,
&header,
&response)) {
printf("Failed to create upload url.\n");
printf("Response code: %ld\n", response_code);
printf("Response:\n");
printf("%s\n", response.c_str());
return false;
}
// Note camel-case rather than underscores.
std::regex upload_url_regex("\"uploadUrl\": \"([^\"]+)\"");
std::regex upload_key_regex("\"uploadKey\": \"([^\"]+)\"");
std::smatch upload_url_match;
if (!std::regex_search(response, upload_url_match, upload_url_regex) ||
upload_url_match.size() != 2) {
printf("Failed to parse create url response.");
printf("Response:\n");
printf("%s\n", response.c_str());
return false;
}
string upload_url = upload_url_match[1].str();
std::smatch upload_key_match;
if (!std::regex_search(response, upload_key_match, upload_key_regex) ||
upload_key_match.size() != 2) {
printf("Failed to parse create url response.");
printf("Response:\n");
printf("%s\n", response.c_str());
return false;
}
string upload_key = upload_key_match[1].str();
uploadUrlResponse->upload_url = upload_url;
uploadUrlResponse->upload_key = upload_key;
return true;
}
// static
CompleteUploadResult SymbolCollectorClient::CompleteUpload(
LibcurlWrapper* libcurl_wrapper,
const string& api_url,
const string& api_key,
const string& upload_key,
const string& debug_file,
const string& debug_id) {
string header, response;
long response_code;
string url = api_url + "/v1/uploads/" + upload_key + ":complete";
if (!api_key.empty()) {
url += "?key=" + api_key;
}
string body =
"{ symbol_id: {"
"debug_file: \"" + debug_file + "\", "
"debug_id: \"" + debug_id + "\" } }";
if (!libcurl_wrapper->SendSimplePostRequest(url,
body,
"application/son",
&response_code,
&header,
&response)) {
printf("Failed to complete upload.\n");
printf("Response code: %ld\n", response_code);
printf("Response:\n");
printf("%s\n", response.c_str());
return CompleteUploadResult::Error;
}
std::regex result_regex("\"result\": \"([^\"]+)\"");
std::smatch result_match;
if (!std::regex_search(response, result_match, result_regex) ||
result_match.size() != 2) {
printf("Failed to parse complete upload response.");
printf("Response:\n");
printf("%s\n", response.c_str());
return CompleteUploadResult::Error;
}
string result = result_match[1].str();
if (result.compare("DUPLICATE_DATA") == 0) {
return CompleteUploadResult::DuplicateData;
}
return CompleteUploadResult::Ok;
}
// static
SymbolStatus SymbolCollectorClient::CheckSymbolStatus(
LibcurlWrapper* libcurl_wrapper,
const string& api_url,
const string& api_key,
const string& debug_file,
const string& debug_id) {
string header, response;
long response_code;
string url = api_url +
"/v1/symbols/" + debug_file + "/" + debug_id + ":checkStatus";
if (!api_key.empty()) {
url += "?key=" + api_key;
}
if (!libcurl_wrapper->SendGetRequest(
url,
&response_code,
&header,
&response)) {
printf("Failed to check symbol status, error message.\n");
printf("Response code: %ld\n", response_code);
printf("Response:\n");
printf("%s\n", response.c_str());
return SymbolStatus::Unknown;
}
std::regex status_regex("\"status\": \"([^\"]+)\"");
std::smatch status_match;
if (!std::regex_search(response, status_match, status_regex) ||
status_match.size() != 2) {
printf("Failed to parse check symbol status response.");
printf("Response:\n");
printf("%s\n", response.c_str());
return SymbolStatus::Unknown;
}
string status = status_match[1].str();
return (status.compare("FOUND") == 0) ?
SymbolStatus::Found :
SymbolStatus::Missing;
}
} // namespace sym_upload
} // namespace google_breakpad

87
src/common/linux/symbol_collector_client.h Normal file
View file

@ -0,0 +1,87 @@
// Copyright (c) 2019, Google Inc.
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
// * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#ifndef COMMON_LINUX_SYMBOL_COLLECTOR_CLIENT_H_
#define COMMON_LINUX_SYMBOL_COLLECTOR_CLIENT_H_
#include <string>
#include "common/linux/libcurl_wrapper.h"
#include "common/using_std_string.h"
namespace google_breakpad {
namespace sym_upload {
struct UploadUrlResponse {
string upload_url;
string upload_key;
};
enum SymbolStatus {
Found,
Missing,
Unknown
};
enum CompleteUploadResult {
Ok,
DuplicateData,
Error
};
// Helper class to communicate with a sym-upload-v2 service over HTTP/REST,
// via libcurl.
class SymbolCollectorClient {
public:
static bool CreateUploadUrl(
LibcurlWrapper* libcurl_wrapper,
const string& api_url,
const string& api_key,
UploadUrlResponse* uploadUrlResponse);
static CompleteUploadResult CompleteUpload(
LibcurlWrapper* libcurl_wrapper,
const string& api_url,
const string& api_key,
const string& upload_key,
const string& debug_file,
const string& debug_id);
static SymbolStatus CheckSymbolStatus(
LibcurlWrapper* libcurl_wrapper,
const string& api_url,
const string& api_key,
const string& debug_file,
const string& debug_id);
};
} // namespace sym_upload
} // namespace google_breakpad
#endif // COMMON_LINUX_SYMBOL_COLLECTOR_CLIENT_H_

151
src/common/linux/symbol_upload.cc
View file

@ -30,15 +30,19 @@
// symbol_upload.cc: implemented google_breakpad::sym_upload::Start, a helper
// function for linux symbol upload tool.
#include "common/linux/http_upload.h"
#include "common/linux/symbol_upload.h"
#include <assert.h>
#include <stdio.h>
#include <functional>
#include <iostream>
#include <vector>
#include "common/linux/http_upload.h"
#include "common/linux/libcurl_wrapper.h"
#include "common/linux/symbol_collector_client.h"
namespace google_breakpad {
namespace sym_upload {
@ -95,21 +99,19 @@ string CompactIdentifier(const string &uuid) {
return result;
}
//=============================================================================
void Start(Options *options) {
// |options| describes the current sym_upload options.
// |module_parts| contains the strings parsed from the MODULE entry of the
// Breakpad symbol file being uploaded.
// |compacted_id| is the debug_id from the MODULE entry of the Breakpad symbol
// file being uploaded, with all hyphens removed.
bool SymUploadV1Start(
const Options& options,
std::vector<string> module_parts,
const string& compacted_id) {
std::map<string, string> parameters;
options->success = false;
std::vector<string> module_parts;
if (!ModuleDataForSymbolFile(options->symbolsPath, &module_parts)) {
fprintf(stderr, "Failed to parse symbol file!\n");
return;
}
string compacted_id = CompactIdentifier(module_parts[3]);
// Add parameters
if (!options->version.empty())
parameters["version"] = options->version;
if (!options.version.empty())
parameters["version"] = options.version;
// MODULE <os> <cpu> <uuid> <module-name>
// 0 1 2 3 4
@ -120,16 +122,16 @@ void Start(Options *options) {
parameters["debug_identifier"] = compacted_id;
std::map<string, string> files;
files["symbol_file"] = options->symbolsPath;
files["symbol_file"] = options.symbolsPath;
string response, error;
long response_code;
bool success = HTTPUpload::SendRequest(options->uploadURLStr,
bool success = HTTPUpload::SendRequest(options.uploadURLStr,
parameters,
files,
options->proxy,
options->proxy_user_pwd,
"",
options.proxy,
options.proxy_user_pwd,
/*ca_certificate_file=*/"",
&response,
&response_code,
&error);
@ -148,7 +150,116 @@ void Start(Options *options) {
} else {
printf("Successfully sent the symbol file.\n");
}
options->success = success;
return success;
}
// |options| describes the current sym_upload options.
// |module_parts| contains the strings parsed from the MODULE entry of the
// Breakpad symbol file being uploaded.
// |compacted_id| is the debug_id from the MODULE entry of the Breakpad symbol
// file being uploaded, with all hyphens removed.
bool SymUploadV2Start(
const Options& options,
std::vector<string> module_parts,
const string& compacted_id) {
string debug_file = module_parts[4];
string debug_id = compacted_id;
google_breakpad::LibcurlWrapper libcurl_wrapper;
if (!libcurl_wrapper.Init()) {
printf("Failed to init google_breakpad::LibcurlWrapper.\n");
return false;
}
if (!options.force) {
SymbolStatus symbolStatus = SymbolCollectorClient::CheckSymbolStatus(
&libcurl_wrapper,
options.uploadURLStr,
options.api_key,
debug_file,
debug_id);
if (symbolStatus == SymbolStatus::Found) {
printf("Symbol file already exists, upload aborted."
" Use \"-f\" to overwrite.\n");
return true;
} else if (symbolStatus == SymbolStatus::Unknown) {
printf("Failed to check for existing symbol.\n");
return false;
}
}
UploadUrlResponse uploadUrlResponse;
if (!SymbolCollectorClient::CreateUploadUrl(
&libcurl_wrapper,
options.uploadURLStr,
options.api_key,
&uploadUrlResponse)) {
printf("Failed to create upload URL.\n");
return false;
}
string signed_url = uploadUrlResponse.upload_url;
string upload_key = uploadUrlResponse.upload_key;
string header;
string response;
long response_code;
if (!libcurl_wrapper.SendPutRequest(signed_url,
options.symbolsPath,
&response_code,
&header,
&response)) {
printf("Failed to send symbol file.\n");
printf("Response code: %ld\n", response_code);
printf("Response:\n");
printf("%s\n", response.c_str());
return false;
} else if (response_code == 0) {
printf("Failed to send symbol file: No response code\n");
return false;
} else if (response_code != 200) {
printf("Failed to send symbol file: Response code %ld\n", response_code);
printf("Response:\n");
printf("%s\n", response.c_str());
return false;
}
CompleteUploadResult completeUploadResult =
SymbolCollectorClient::CompleteUpload(&libcurl_wrapper,
options.uploadURLStr,
options.api_key,
upload_key,
debug_file,
debug_id);
if (completeUploadResult == CompleteUploadResult::Error) {
printf("Failed to complete upload.\n");
return false;
} else if (completeUploadResult == CompleteUploadResult::DuplicateData) {
printf("Uploaded file checksum matched existing file checksum,"
" no change necessary.\n");
} else {
printf("Successfully sent the symbol file.\n");
}
return true;
}
//=============================================================================
void Start(Options* options) {
std::vector<string> module_parts;
if (!ModuleDataForSymbolFile(options->symbolsPath, &module_parts)) {
fprintf(stderr, "Failed to parse symbol file!\n");
return;
}
const string compacted_id = CompactIdentifier(module_parts[3]);
if (options->upload_protocol == UploadProtocol::SYM_UPLOAD_V2) {
options->success = SymUploadV2Start(*options, module_parts, compacted_id);
} else {
options->success = SymUploadV1Start(*options, module_parts, compacted_id);
}
}
} // namespace sym_upload

14
src/common/linux/symbol_upload.h
View file

@ -41,14 +41,24 @@
namespace google_breakpad {
namespace sym_upload {
typedef struct {
enum class UploadProtocol {
SYM_UPLOAD_V1,
SYM_UPLOAD_V2,
};
struct Options {
Options() : upload_protocol(UploadProtocol::SYM_UPLOAD_V1), force(false) {}
string symbolsPath;
string uploadURLStr;
string proxy;
string proxy_user_pwd;
string version;
bool success;
} Options;
UploadProtocol upload_protocol;
bool force;
string api_key;
};
// Starts upload to symbol server with options.
void Start(Options* options);

41
src/tools/linux/symupload/sym_upload.cc
View file

@ -41,25 +41,43 @@
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include "common/linux/symbol_upload.h"
using google_breakpad::sym_upload::UploadProtocol;
using google_breakpad::sym_upload::Options;
//=============================================================================
static void
Usage(int argc, const char *argv[]) {
fprintf(stderr, "Submit symbol information.\n");
fprintf(stderr, "Usage: %s [options...] <symbols> <upload-URL>\n", argv[0]);
fprintf(stderr, "Usage: %s [options...] <symbol-file> <upload-URL>\n",
argv[0]);
fprintf(stderr, "Options:\n");
fprintf(stderr, "<symbols> should be created by using the dump_syms tool.\n");
fprintf(stderr, "<symbol-file> should be created by using the dump_syms"
"tool.\n");
fprintf(stderr, "<upload-URL> is the destination for the upload\n");
fprintf(stderr, "-p:\t <protocol> One of ['sym-upload-v1',"
" 'sym-upload-v2'], defaults to 'sym-upload-v1'.\n");
fprintf(stderr, "-k:\t <API-key> A secret used to authenticate with the"
" API [Only supported when using 'sym-upload-v2' protocol].\n");
fprintf(stderr, "-f:\t Force symbol upload if already exists [Only"
" supported when using 'sym-upload-v2' protocol].\n");
fprintf(stderr, "-v:\t Version information (e.g., 1.2.3.4)\n");
fprintf(stderr, "-x:\t <host[:port]> Use HTTP proxy on given port\n");
fprintf(stderr, "-u:\t <user[:password]> Set proxy user and password\n");
fprintf(stderr, "-h:\t Usage\n");
fprintf(stderr, "-?:\t Usage\n");
fprintf(stderr, "\n");
fprintf(stderr, "Examples:\n");
fprintf(stderr, " With 'sym-upload-v1':\n");
fprintf(stderr, " %s path/to/symbol_file http://myuploadserver\n",
argv[0]);
fprintf(stderr, " With 'sym-upload-v2':\n");
fprintf(stderr, " %s -p sym-upload-v2 -k mysecret123! "
"path/to/symbol_file http://myuploadserver\n", argv[0]);
}
//=============================================================================
@ -68,7 +86,7 @@ SetupOptions(int argc, const char *argv[], Options *options) {
extern int optind;
int ch;
while ((ch = getopt(argc, (char * const *)argv, "u:v:x:h?")) != -1) {
while ((ch = getopt(argc, (char * const *)argv, "u:v:x:p:k:hf?")) != -1) {
switch (ch) {
case 'h':
case '?':
@ -84,6 +102,23 @@ SetupOptions(int argc, const char *argv[], Options *options) {
case 'x':
options->proxy = optarg;
break;
case 'p':
if (strcmp(optarg, "sym-upload-v2") == 0) {
options->upload_protocol = UploadProtocol::SYM_UPLOAD_V2;
} else if (strcmp(optarg, "sym-upload-v1") == 0) {
options->upload_protocol = UploadProtocol::SYM_UPLOAD_V1;
} else {
fprintf(stderr, "Invalid protocol '%c'\n", optarg);
Usage(argc, argv);
exit(1);
}
break;
case 'k':
options->api_key = optarg;
break;
case 'f':
options->force = true;
break;
default:
fprintf(stderr, "Invalid option '%c'\n", ch);