Add optional new symbol upload API to sym_upload.
Change-Id: I6a49e9f4a699fa6f5f8e9f0fc86afb4cb342a442 Reviewed-on: https://chromium-review.googlesource.com/c/breakpad/breakpad/+/1422400 Reviewed-by: Mark Mentovai <mark@chromium.org> Reviewed-by: Ivan Penkov <ivanpe@chromium.org> Reviewed-by: Mike Frysinger <vapier@chromium.org>
This commit is contained in:
Nelson Billing
parent
216cea7bca
commit
bbad9f255d
11 changed files with 880 additions and 99 deletions
|
|
@ -615,6 +615,10 @@ src_tools_linux_symupload_minidump_upload_LDADD = -ldl
|
||||||
src_tools_linux_symupload_sym_upload_SOURCES = \
|
src_tools_linux_symupload_sym_upload_SOURCES = \
|
||||||
src/common/linux/http_upload.cc \
|
src/common/linux/http_upload.cc \
|
||||||
src/common/linux/http_upload.h \
|
src/common/linux/http_upload.h \
|
||||||
|
src/common/linux/libcurl_wrapper.cc \
|
||||||
|
src/common/linux/libcurl_wrapper.h \
|
||||||
|
src/common/linux/symbol_collector_client.cc \
|
||||||
|
src/common/linux/symbol_collector_client.h \
|
||||||
src/common/linux/symbol_upload.cc \
|
src/common/linux/symbol_upload.cc \
|
||||||
src/common/linux/symbol_upload.h \
|
src/common/linux/symbol_upload.h \
|
||||||
src/tools/linux/symupload/sym_upload.cc
|
src/tools/linux/symupload/sym_upload.cc
|
||||||
|
|
|
||||||
14
Makefile.in
14
Makefile.in
|
|
@ -1495,10 +1495,16 @@ src_tools_linux_symupload_minidump_upload_OBJECTS = \
|
||||||
src_tools_linux_symupload_minidump_upload_DEPENDENCIES =
|
src_tools_linux_symupload_minidump_upload_DEPENDENCIES =
|
||||||
am__src_tools_linux_symupload_sym_upload_SOURCES_DIST = \
|
am__src_tools_linux_symupload_sym_upload_SOURCES_DIST = \
|
||||||
src/common/linux/http_upload.cc src/common/linux/http_upload.h \
|
src/common/linux/http_upload.cc src/common/linux/http_upload.h \
|
||||||
|
src/common/linux/libcurl_wrapper.cc \
|
||||||
|
src/common/linux/libcurl_wrapper.h \
|
||||||
|
src/common/linux/symbol_collector_client.cc \
|
||||||
|
src/common/linux/symbol_collector_client.h \
|
||||||
src/common/linux/symbol_upload.cc \
|
src/common/linux/symbol_upload.cc \
|
||||||
src/common/linux/symbol_upload.h \
|
src/common/linux/symbol_upload.h \
|
||||||
src/tools/linux/symupload/sym_upload.cc
|
src/tools/linux/symupload/sym_upload.cc
|
||||||
@DISABLE_TOOLS_FALSE@@LINUX_HOST_TRUE@am_src_tools_linux_symupload_sym_upload_OBJECTS = src/common/linux/http_upload.$(OBJEXT) \
|
@DISABLE_TOOLS_FALSE@@LINUX_HOST_TRUE@am_src_tools_linux_symupload_sym_upload_OBJECTS = src/common/linux/http_upload.$(OBJEXT) \
|
||||||
|
@DISABLE_TOOLS_FALSE@@LINUX_HOST_TRUE@ src/common/linux/libcurl_wrapper.$(OBJEXT) \
|
||||||
|
@DISABLE_TOOLS_FALSE@@LINUX_HOST_TRUE@ src/common/linux/symbol_collector_client.$(OBJEXT) \
|
||||||
@DISABLE_TOOLS_FALSE@@LINUX_HOST_TRUE@ src/common/linux/symbol_upload.$(OBJEXT) \
|
@DISABLE_TOOLS_FALSE@@LINUX_HOST_TRUE@ src/common/linux/symbol_upload.$(OBJEXT) \
|
||||||
@DISABLE_TOOLS_FALSE@@LINUX_HOST_TRUE@ src/tools/linux/symupload/sym_upload.$(OBJEXT)
|
@DISABLE_TOOLS_FALSE@@LINUX_HOST_TRUE@ src/tools/linux/symupload/sym_upload.$(OBJEXT)
|
||||||
src_tools_linux_symupload_sym_upload_OBJECTS = \
|
src_tools_linux_symupload_sym_upload_OBJECTS = \
|
||||||
|
|
@ -2477,6 +2483,10 @@ TESTS = $(check_PROGRAMS) $(check_SCRIPTS)
|
||||||
@DISABLE_TOOLS_FALSE@@LINUX_HOST_TRUE@src_tools_linux_symupload_sym_upload_SOURCES = \
|
@DISABLE_TOOLS_FALSE@@LINUX_HOST_TRUE@src_tools_linux_symupload_sym_upload_SOURCES = \
|
||||||
@DISABLE_TOOLS_FALSE@@LINUX_HOST_TRUE@ src/common/linux/http_upload.cc \
|
@DISABLE_TOOLS_FALSE@@LINUX_HOST_TRUE@ src/common/linux/http_upload.cc \
|
||||||
@DISABLE_TOOLS_FALSE@@LINUX_HOST_TRUE@ src/common/linux/http_upload.h \
|
@DISABLE_TOOLS_FALSE@@LINUX_HOST_TRUE@ src/common/linux/http_upload.h \
|
||||||
|
@DISABLE_TOOLS_FALSE@@LINUX_HOST_TRUE@ src/common/linux/libcurl_wrapper.cc \
|
||||||
|
@DISABLE_TOOLS_FALSE@@LINUX_HOST_TRUE@ src/common/linux/libcurl_wrapper.h \
|
||||||
|
@DISABLE_TOOLS_FALSE@@LINUX_HOST_TRUE@ src/common/linux/symbol_collector_client.cc \
|
||||||
|
@DISABLE_TOOLS_FALSE@@LINUX_HOST_TRUE@ src/common/linux/symbol_collector_client.h \
|
||||||
@DISABLE_TOOLS_FALSE@@LINUX_HOST_TRUE@ src/common/linux/symbol_upload.cc \
|
@DISABLE_TOOLS_FALSE@@LINUX_HOST_TRUE@ src/common/linux/symbol_upload.cc \
|
||||||
@DISABLE_TOOLS_FALSE@@LINUX_HOST_TRUE@ src/common/linux/symbol_upload.h \
|
@DISABLE_TOOLS_FALSE@@LINUX_HOST_TRUE@ src/common/linux/symbol_upload.h \
|
||||||
@DISABLE_TOOLS_FALSE@@LINUX_HOST_TRUE@ src/tools/linux/symupload/sym_upload.cc
|
@DISABLE_TOOLS_FALSE@@LINUX_HOST_TRUE@ src/tools/linux/symupload/sym_upload.cc
|
||||||
|
|
@ -4726,6 +4736,9 @@ src/tools/linux/symupload/minidump_upload.$(OBJEXT): \
|
||||||
src/tools/linux/symupload/minidump_upload$(EXEEXT): $(src_tools_linux_symupload_minidump_upload_OBJECTS) $(src_tools_linux_symupload_minidump_upload_DEPENDENCIES) $(EXTRA_src_tools_linux_symupload_minidump_upload_DEPENDENCIES) src/tools/linux/symupload/$(am__dirstamp)
|
src/tools/linux/symupload/minidump_upload$(EXEEXT): $(src_tools_linux_symupload_minidump_upload_OBJECTS) $(src_tools_linux_symupload_minidump_upload_DEPENDENCIES) $(EXTRA_src_tools_linux_symupload_minidump_upload_DEPENDENCIES) src/tools/linux/symupload/$(am__dirstamp)
|
||||||
@rm -f src/tools/linux/symupload/minidump_upload$(EXEEXT)
|
@rm -f src/tools/linux/symupload/minidump_upload$(EXEEXT)
|
||||||
$(AM_V_CXXLD)$(CXXLINK) $(src_tools_linux_symupload_minidump_upload_OBJECTS) $(src_tools_linux_symupload_minidump_upload_LDADD) $(LIBS)
|
$(AM_V_CXXLD)$(CXXLINK) $(src_tools_linux_symupload_minidump_upload_OBJECTS) $(src_tools_linux_symupload_minidump_upload_LDADD) $(LIBS)
|
||||||
|
src/common/linux/symbol_collector_client.$(OBJEXT): \
|
||||||
|
src/common/linux/$(am__dirstamp) \
|
||||||
|
src/common/linux/$(DEPDIR)/$(am__dirstamp)
|
||||||
src/common/linux/symbol_upload.$(OBJEXT): \
|
src/common/linux/symbol_upload.$(OBJEXT): \
|
||||||
src/common/linux/$(am__dirstamp) \
|
src/common/linux/$(am__dirstamp) \
|
||||||
src/common/linux/$(DEPDIR)/$(am__dirstamp)
|
src/common/linux/$(DEPDIR)/$(am__dirstamp)
|
||||||
|
|
@ -4996,6 +5009,7 @@ distclean-compile:
|
||||||
@AMDEP_TRUE@@am__include@ @am__quote@src/common/linux/$(DEPDIR)/src_tools_linux_dump_syms_dump_syms-linux_libc_support.Po@am__quote@
|
@AMDEP_TRUE@@am__include@ @am__quote@src/common/linux/$(DEPDIR)/src_tools_linux_dump_syms_dump_syms-linux_libc_support.Po@am__quote@
|
||||||
@AMDEP_TRUE@@am__include@ @am__quote@src/common/linux/$(DEPDIR)/src_tools_linux_dump_syms_dump_syms-memory_mapped_file.Po@am__quote@
|
@AMDEP_TRUE@@am__include@ @am__quote@src/common/linux/$(DEPDIR)/src_tools_linux_dump_syms_dump_syms-memory_mapped_file.Po@am__quote@
|
||||||
@AMDEP_TRUE@@am__include@ @am__quote@src/common/linux/$(DEPDIR)/src_tools_linux_dump_syms_dump_syms-safe_readlink.Po@am__quote@
|
@AMDEP_TRUE@@am__include@ @am__quote@src/common/linux/$(DEPDIR)/src_tools_linux_dump_syms_dump_syms-safe_readlink.Po@am__quote@
|
||||||
|
@AMDEP_TRUE@@am__include@ @am__quote@src/common/linux/$(DEPDIR)/symbol_collector_client.Po@am__quote@
|
||||||
@AMDEP_TRUE@@am__include@ @am__quote@src/common/linux/$(DEPDIR)/symbol_upload.Po@am__quote@
|
@AMDEP_TRUE@@am__include@ @am__quote@src/common/linux/$(DEPDIR)/symbol_upload.Po@am__quote@
|
||||||
@AMDEP_TRUE@@am__include@ @am__quote@src/common/linux/tests/$(DEPDIR)/src_client_linux_linux_client_unittest_shlib-crash_generator.Po@am__quote@
|
@AMDEP_TRUE@@am__include@ @am__quote@src/common/linux/tests/$(DEPDIR)/src_client_linux_linux_client_unittest_shlib-crash_generator.Po@am__quote@
|
||||||
@AMDEP_TRUE@@am__include@ @am__quote@src/common/linux/tests/$(DEPDIR)/src_common_dumper_unittest-crash_generator.Po@am__quote@
|
@AMDEP_TRUE@@am__include@ @am__quote@src/common/linux/tests/$(DEPDIR)/src_common_dumper_unittest-crash_generator.Po@am__quote@
|
||||||
|
|
|
||||||
214
docs/sym_upload_v2_protocol.md
Normal file
214
docs/sym_upload_v2_protocol.md
Normal file
|
|
@ -0,0 +1,214 @@
|
||||||
|
# Introduction
|
||||||
|
|
||||||
|
The `sym_upload` tool is able to operate in `sym-upload-v2` protocol mode, in
|
||||||
|
addition to the legacy protocol (which will be referred to as `sym-upload-v1`
|
||||||
|
for the rest of this document). For now `sym-upload-v2` is HTTP/REST-based but
|
||||||
|
it could be extended to operate over gRPC instead, in the future.
|
||||||
|
|
||||||
|
# Table of Contents
|
||||||
|
* [Why](#why)
|
||||||
|
* [How](#how)
|
||||||
|
* [Uploading](#uploading)
|
||||||
|
* [Uploading with `sym_upload`](#uploading-with-sym_upload)
|
||||||
|
* [Uploading with curl](#uploading-with-curl)
|
||||||
|
* [Serving the `sym-upload-v2` protocol](#serving-the-sym-upload-v2-protocol)
|
||||||
|
* [Authenticate using `key`](#authenticate-using-key)
|
||||||
|
* [Symbol `checkStatus`](#symbol-checkstatus)
|
||||||
|
* [Upload `create`](#upload-create)
|
||||||
|
* [Uploading the symbol file](#uploading-the-symbol-file)
|
||||||
|
* [Upload complete](#upload-complete)
|
||||||
|
|
||||||
|
|
||||||
|
# Why
|
||||||
|
|
||||||
|
Using `sym_upload` in `sym-upload-v2` protocol mode has the following features
|
||||||
|
beyond `sym-upload-v1`:
|
||||||
|
* Authentication via `key` (arbitrary secret).
|
||||||
|
* Symbol identifier (product of `debug_file` and `debug_id`, as recorded in
|
||||||
|
output from `dump_syms`) can be checked against existing symbol information on
|
||||||
|
server. If it's present, then the upload is skipped entirely.
|
||||||
|
|
||||||
|
# How
|
||||||
|
|
||||||
|
## Uploading
|
||||||
|
|
||||||
|
### Uploading with `sym_upload`
|
||||||
|
|
||||||
|
Uploading in `sym-upload-v2` protocol mode is easy. Invoke `sym_upload` like
|
||||||
|
```
|
||||||
|
$ ./sym_upload -p sym-upload-v2 [-k <API-key>] <symbol-file> <API-URL>
|
||||||
|
```
|
||||||
|
|
||||||
|
Where `symbol-file` is a symbol file created by `dump_syms`, `API-URL` is the
|
||||||
|
URL of your `sym-upload-v2` API service (see next section for details), and
|
||||||
|
`API-key` is a secret known to your uploader and server.
|
||||||
|
|
||||||
|
For more options see `sym_upload --help`.
|
||||||
|
|
||||||
|
### Uploading with curl
|
||||||
|
|
||||||
|
As an example, if:
|
||||||
|
* Your API's URL was "https://sym-upload-api".
|
||||||
|
* Your service has assigned you `key` "myfancysecret123".
|
||||||
|
* You wanted to upload the symbol file at "path/to/file_name", with
|
||||||
|
`debug_file` being "file_name" and `debug_id` being
|
||||||
|
"123123123123123123123123123". Normally you would read these values from
|
||||||
|
"path/to/file_name", which in turn was generated by `dump_syms`.
|
||||||
|
|
||||||
|
Then you might run:
|
||||||
|
```
|
||||||
|
$ curl https://sym-upload-api/symbols/file_name/123123123123123123123123123:checkStatus?key=myfancysecret123
|
||||||
|
```
|
||||||
|
|
||||||
|
And, upon seeing that this `debug_file`/`debug_id` combo is missing from symbol
|
||||||
|
storage then you could run:
|
||||||
|
```
|
||||||
|
$ curl --request POST https://sym-upload-api/uploads:create?key=myfancysecret123
|
||||||
|
```
|
||||||
|
|
||||||
|
Which returns `upload_url` "https://upload-server/42?creds=shhhhh" and
|
||||||
|
`upload_key` "42". Next you upload the file directly like:
|
||||||
|
```
|
||||||
|
$ curl -T path/to/file_name "https://upload-server/42?creds=shhhhh"
|
||||||
|
```
|
||||||
|
|
||||||
|
Once the HTTP PUT is complete, run:
|
||||||
|
```
|
||||||
|
$ curl --header "Content-Type: application/json" \
|
||||||
|
--request POST \
|
||||||
|
--data '{symbol_id:{"debugFile":"file_name",'\
|
||||||
|
'"debugId":"123123123123123123123123123"}}' \
|
||||||
|
https://sym-upload-api/uploads/42:complete?key=myfancysecret123
|
||||||
|
```
|
||||||
|
|
||||||
|
### Serving the `sym-upload-v2` Protocol
|
||||||
|
|
||||||
|
The protocol is currently defined only in HTTP/REST. There are three necessary
|
||||||
|
REST operations to implement in your service:
|
||||||
|
* `/symbols/<debug_file>/<debug_id>:checkStatus?key=<key>`
|
||||||
|
* `/uploads:create?key=<key>`
|
||||||
|
* `/uploads/<upload_key>:complete?key=<key>`
|
||||||
|
|
||||||
|
#### Authenticate Using `key`
|
||||||
|
|
||||||
|
The query string arg `key` contains some secret that both the uploader and
|
||||||
|
server understand. It is up to the service implementer to decide on what
|
||||||
|
constitutes a valid `key`, how the uploader acquires one, and how to handle
|
||||||
|
requests made with invalid ones.
|
||||||
|
|
||||||
|
#### Symbol `checkStatus`
|
||||||
|
|
||||||
|
```
|
||||||
|
/symbols/<debug_file>/<debug_id>:checkStatus?key=<key>
|
||||||
|
```
|
||||||
|
|
||||||
|
This operation expects an empty (or no) JSON payload in the request.
|
||||||
|
|
||||||
|
This operation should return the status of the symbol file uniquely identified
|
||||||
|
by the given `debug_file` and `debug_id`. JSON schema:
|
||||||
|
```
|
||||||
|
{
|
||||||
|
"type": object",
|
||||||
|
"properties": {
|
||||||
|
"status": {
|
||||||
|
"type": "string",
|
||||||
|
"enum": ["STATUS_UNSPECIFIED", "MISING", "FOUND"],
|
||||||
|
"required": true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Where `MISSING` denotes that the symbol file does not exist on the server and
|
||||||
|
`FOUND` denotes that the symbol file exists on the server.
|
||||||
|
|
||||||
|
#### Upload `create`
|
||||||
|
|
||||||
|
```
|
||||||
|
/uploads:create?key=<key>
|
||||||
|
```
|
||||||
|
|
||||||
|
This operation expects an empty (or no) JSON payload in the request.
|
||||||
|
|
||||||
|
This operation should return a URL that uploader can HTTP PUT their symbol file
|
||||||
|
to, along with an "upload key" that can be used to notify the service once the
|
||||||
|
file upload is completed. JSON schema:
|
||||||
|
```
|
||||||
|
{
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"upload_url": {
|
||||||
|
"type: "string",
|
||||||
|
"required": true
|
||||||
|
},
|
||||||
|
"upload_key": {
|
||||||
|
"type": "string",
|
||||||
|
"required": true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Since this REST API operation can be authenticated via the `key` query string
|
||||||
|
arg, the service can return a URL that encodes permission delegation to the
|
||||||
|
upload endpoint resource and thereby constrain the ability to upload to those
|
||||||
|
with valid `key`s.
|
||||||
|
|
||||||
|
#### Uploading the Symbol File
|
||||||
|
|
||||||
|
Note that the actual symbol upload step is _not_ part of the REST API. The
|
||||||
|
upload URL obtained in the above operation is meant to be used as the endpoint
|
||||||
|
for a normal HTTP PUT request for the contents of the symbol file. Once that
|
||||||
|
HTTP PUT request is completed use the upload `complete` operation.
|
||||||
|
|
||||||
|
#### Upload `complete`
|
||||||
|
|
||||||
|
```
|
||||||
|
/uploads/<upload_key>:complete?key=<key>
|
||||||
|
```
|
||||||
|
|
||||||
|
This operation expects a JSON payload in the HTTP request body with the
|
||||||
|
following schema:
|
||||||
|
```
|
||||||
|
{
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"symbol_id": {
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"debug_file": {
|
||||||
|
"type": "string",
|
||||||
|
"required": true
|
||||||
|
},
|
||||||
|
"debug_id": {
|
||||||
|
"type": "string",
|
||||||
|
"required": true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
This operation should cause the symbol storage back-end (however implemented)
|
||||||
|
to consume the symbol file identified by `upload_key`. It is up to the service
|
||||||
|
implementation to decide how uploads are assigned `upload_key`s and how to
|
||||||
|
retrieve a completed upload by its `upload_key`. If the symbol file cannot be
|
||||||
|
found, is malformed, or the operation cannot be completed for any other reason
|
||||||
|
then an HTTP error will be returned. JSON schema of non-error responses:
|
||||||
|
```
|
||||||
|
{
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"result": {
|
||||||
|
"type": string,
|
||||||
|
"enum": ["RESULT_UNSPECIFIED", "OK", "DUPLICATE_DATA"],
|
||||||
|
"required": true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Where `OK` denotes that the symbol storage was updated with the new symbol file
|
||||||
|
and `DUPLICATE_DATA` denotes that the symbol file data was identical to data
|
||||||
|
already in symbol storage and therefore nothing changed.
|
||||||
|
|
@ -74,8 +74,8 @@
|
||||||
'dwarf/dwarf2reader.cc',
|
'dwarf/dwarf2reader.cc',
|
||||||
'dwarf/dwarf2reader.h',
|
'dwarf/dwarf2reader.h',
|
||||||
'dwarf/dwarf2reader_test_common.h',
|
'dwarf/dwarf2reader_test_common.h',
|
||||||
'dwarf/elf_reader.cc',
|
'dwarf/elf_reader.cc',
|
||||||
'dwarf/elf_reader.h',
|
'dwarf/elf_reader.h',
|
||||||
'dwarf/functioninfo.cc',
|
'dwarf/functioninfo.cc',
|
||||||
'dwarf/functioninfo.h',
|
'dwarf/functioninfo.h',
|
||||||
'dwarf/line_state_machine.h',
|
'dwarf/line_state_machine.h',
|
||||||
|
|
@ -118,6 +118,8 @@
|
||||||
'linux/memory_mapped_file.h',
|
'linux/memory_mapped_file.h',
|
||||||
'linux/safe_readlink.cc',
|
'linux/safe_readlink.cc',
|
||||||
'linux/safe_readlink.h',
|
'linux/safe_readlink.h',
|
||||||
|
'linux/symbol_collector_client.cc',
|
||||||
|
'linux/symbol_collector_client.h',
|
||||||
'linux/synth_elf.cc',
|
'linux/synth_elf.cc',
|
||||||
'linux/synth_elf.h',
|
'linux/synth_elf.h',
|
||||||
'long_string_dictionary.cc',
|
'long_string_dictionary.cc',
|
||||||
|
|
|
||||||
|
|
@ -38,32 +38,24 @@
|
||||||
namespace google_breakpad {
|
namespace google_breakpad {
|
||||||
LibcurlWrapper::LibcurlWrapper()
|
LibcurlWrapper::LibcurlWrapper()
|
||||||
: init_ok_(false),
|
: init_ok_(false),
|
||||||
formpost_(NULL),
|
curl_lib_(nullptr),
|
||||||
lastptr_(NULL),
|
last_curl_error_(""),
|
||||||
headerlist_(NULL) {
|
curl_(nullptr),
|
||||||
curl_lib_ = dlopen("libcurl.so", RTLD_NOW);
|
formpost_(nullptr),
|
||||||
if (!curl_lib_) {
|
lastptr_(nullptr),
|
||||||
curl_lib_ = dlopen("libcurl.so.4", RTLD_NOW);
|
headerlist_(nullptr) {}
|
||||||
}
|
|
||||||
if (!curl_lib_) {
|
|
||||||
curl_lib_ = dlopen("libcurl.so.3", RTLD_NOW);
|
|
||||||
}
|
|
||||||
if (!curl_lib_) {
|
|
||||||
std::cout << "Could not find libcurl via dlopen";
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
std::cout << "LibcurlWrapper init succeeded";
|
|
||||||
init_ok_ = true;
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
LibcurlWrapper::~LibcurlWrapper() {}
|
LibcurlWrapper::~LibcurlWrapper() {
|
||||||
|
if (init_ok_) {
|
||||||
|
(*easy_cleanup_)(curl_);
|
||||||
|
dlclose(curl_lib_);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
bool LibcurlWrapper::SetProxy(const string& proxy_host,
|
bool LibcurlWrapper::SetProxy(const string& proxy_host,
|
||||||
const string& proxy_userpwd) {
|
const string& proxy_userpwd) {
|
||||||
if (!init_ok_) {
|
if (!CheckInit()) return false;
|
||||||
return false;
|
|
||||||
}
|
|
||||||
// Set proxy information if necessary.
|
// Set proxy information if necessary.
|
||||||
if (!proxy_host.empty()) {
|
if (!proxy_host.empty()) {
|
||||||
(*easy_setopt_)(curl_, CURLOPT_PROXY, proxy_host.c_str());
|
(*easy_setopt_)(curl_, CURLOPT_PROXY, proxy_host.c_str());
|
||||||
|
|
@ -83,9 +75,8 @@ bool LibcurlWrapper::SetProxy(const string& proxy_host,
|
||||||
|
|
||||||
bool LibcurlWrapper::AddFile(const string& upload_file_path,
|
bool LibcurlWrapper::AddFile(const string& upload_file_path,
|
||||||
const string& basename) {
|
const string& basename) {
|
||||||
if (!init_ok_) {
|
if (!CheckInit()) return false;
|
||||||
return false;
|
|
||||||
}
|
|
||||||
std::cout << "Adding " << upload_file_path << " to form upload.";
|
std::cout << "Adding " << upload_file_path << " to form upload.";
|
||||||
// Add form file.
|
// Add form file.
|
||||||
(*formadd_)(&formpost_, &lastptr_,
|
(*formadd_)(&formpost_, &lastptr_,
|
||||||
|
|
@ -110,10 +101,11 @@ static size_t WriteCallback(void *ptr, size_t size,
|
||||||
|
|
||||||
bool LibcurlWrapper::SendRequest(const string& url,
|
bool LibcurlWrapper::SendRequest(const string& url,
|
||||||
const std::map<string, string>& parameters,
|
const std::map<string, string>& parameters,
|
||||||
int* http_status_code,
|
long* http_status_code,
|
||||||
string* http_header_data,
|
string* http_header_data,
|
||||||
string* http_response_data) {
|
string* http_response_data) {
|
||||||
(*easy_setopt_)(curl_, CURLOPT_URL, url.c_str());
|
if (!CheckInit()) return false;
|
||||||
|
|
||||||
std::map<string, string>::const_iterator iter = parameters.begin();
|
std::map<string, string>::const_iterator iter = parameters.begin();
|
||||||
for (; iter != parameters.end(); ++iter)
|
for (; iter != parameters.end(); ++iter)
|
||||||
(*formadd_)(&formpost_, &lastptr_,
|
(*formadd_)(&formpost_, &lastptr_,
|
||||||
|
|
@ -122,55 +114,79 @@ bool LibcurlWrapper::SendRequest(const string& url,
|
||||||
CURLFORM_END);
|
CURLFORM_END);
|
||||||
|
|
||||||
(*easy_setopt_)(curl_, CURLOPT_HTTPPOST, formpost_);
|
(*easy_setopt_)(curl_, CURLOPT_HTTPPOST, formpost_);
|
||||||
if (http_response_data != NULL) {
|
|
||||||
http_response_data->clear();
|
return SendRequestInner(url, http_status_code, http_header_data,
|
||||||
(*easy_setopt_)(curl_, CURLOPT_WRITEFUNCTION, WriteCallback);
|
http_response_data);
|
||||||
(*easy_setopt_)(curl_, CURLOPT_WRITEDATA,
|
}
|
||||||
reinterpret_cast<void *>(http_response_data));
|
|
||||||
}
|
bool LibcurlWrapper::SendGetRequest(const string& url,
|
||||||
if (http_header_data != NULL) {
|
long* http_status_code,
|
||||||
http_header_data->clear();
|
string* http_header_data,
|
||||||
(*easy_setopt_)(curl_, CURLOPT_HEADERFUNCTION, WriteCallback);
|
string* http_response_data) {
|
||||||
(*easy_setopt_)(curl_, CURLOPT_HEADERDATA,
|
if (!CheckInit()) return false;
|
||||||
reinterpret_cast<void *>(http_header_data));
|
|
||||||
|
(*easy_setopt_)(curl_, CURLOPT_HTTPGET, 1L);
|
||||||
|
|
||||||
|
return SendRequestInner(url, http_status_code, http_header_data,
|
||||||
|
http_response_data);
|
||||||
|
}
|
||||||
|
|
||||||
|
bool LibcurlWrapper::SendPutRequest(const string& url,
|
||||||
|
const string& path,
|
||||||
|
long* http_status_code,
|
||||||
|
string* http_header_data,
|
||||||
|
string* http_response_data) {
|
||||||
|
if (!CheckInit()) return false;
|
||||||
|
|
||||||
|
FILE* file = fopen(path.c_str(), "rb");
|
||||||
|
(*easy_setopt_)(curl_, CURLOPT_UPLOAD, 1L);
|
||||||
|
(*easy_setopt_)(curl_, CURLOPT_PUT, 1L);
|
||||||
|
(*easy_setopt_)(curl_, CURLOPT_READDATA, file);
|
||||||
|
|
||||||
|
bool success = SendRequestInner(url, http_status_code, http_header_data,
|
||||||
|
http_response_data);
|
||||||
|
|
||||||
|
fclose(file);
|
||||||
|
return success;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool LibcurlWrapper::SendSimplePostRequest(const string& url,
|
||||||
|
const string& body,
|
||||||
|
const string& content_type,
|
||||||
|
long* http_status_code,
|
||||||
|
string* http_header_data,
|
||||||
|
string* http_response_data) {
|
||||||
|
if (!CheckInit()) return false;
|
||||||
|
|
||||||
|
(*easy_setopt_)(curl_, CURLOPT_POSTFIELDSIZE, body.size());
|
||||||
|
(*easy_setopt_)(curl_, CURLOPT_COPYPOSTFIELDS, body.c_str());
|
||||||
|
|
||||||
|
if (!content_type.empty()) {
|
||||||
|
string content_type_header = "Content-Type: " + content_type;
|
||||||
|
headerlist_ = (*slist_append_)(
|
||||||
|
headerlist_,
|
||||||
|
content_type_header.c_str());
|
||||||
}
|
}
|
||||||
|
|
||||||
CURLcode err_code = CURLE_OK;
|
return SendRequestInner(url, http_status_code, http_header_data,
|
||||||
err_code = (*easy_perform_)(curl_);
|
http_response_data);
|
||||||
easy_strerror_ = reinterpret_cast<const char* (*)(CURLcode)>
|
|
||||||
(dlsym(curl_lib_, "curl_easy_strerror"));
|
|
||||||
|
|
||||||
if (http_status_code != NULL) {
|
|
||||||
(*easy_getinfo_)(curl_, CURLINFO_RESPONSE_CODE, http_status_code);
|
|
||||||
}
|
|
||||||
|
|
||||||
#ifndef NDEBUG
|
|
||||||
if (err_code != CURLE_OK)
|
|
||||||
fprintf(stderr, "Failed to send http request to %s, error: %s\n",
|
|
||||||
url.c_str(),
|
|
||||||
(*easy_strerror_)(err_code));
|
|
||||||
#endif
|
|
||||||
if (headerlist_ != NULL) {
|
|
||||||
(*slist_free_all_)(headerlist_);
|
|
||||||
}
|
|
||||||
|
|
||||||
(*easy_cleanup_)(curl_);
|
|
||||||
if (formpost_ != NULL) {
|
|
||||||
(*formfree_)(formpost_);
|
|
||||||
}
|
|
||||||
|
|
||||||
return err_code == CURLE_OK;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
bool LibcurlWrapper::Init() {
|
bool LibcurlWrapper::Init() {
|
||||||
if (!init_ok_) {
|
curl_lib_ = dlopen("libcurl.so", RTLD_NOW);
|
||||||
std::cout << "Init_OK was not true in LibcurlWrapper::Init(), check earlier log messages";
|
if (!curl_lib_) {
|
||||||
|
curl_lib_ = dlopen("libcurl.so.4", RTLD_NOW);
|
||||||
|
}
|
||||||
|
if (!curl_lib_) {
|
||||||
|
curl_lib_ = dlopen("libcurl.so.3", RTLD_NOW);
|
||||||
|
}
|
||||||
|
if (!curl_lib_) {
|
||||||
|
std::cout << "Could not find libcurl via dlopen";
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!SetFunctionPointers()) {
|
if (!SetFunctionPointers()) {
|
||||||
std::cout << "Could not find function pointers";
|
std::cout << "Could not find function pointers";
|
||||||
init_ok_ = false;
|
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -184,11 +200,7 @@ bool LibcurlWrapper::Init() {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Disable 100-continue header.
|
init_ok_ = true;
|
||||||
char buf[] = "Expect:";
|
|
||||||
|
|
||||||
headerlist_ = (*slist_append_)(headerlist_, buf);
|
|
||||||
(*easy_setopt_)(curl_, CURLOPT_HTTPHEADER, headerlist_);
|
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -228,6 +240,10 @@ bool LibcurlWrapper::SetFunctionPointers() {
|
||||||
"curl_easy_getinfo",
|
"curl_easy_getinfo",
|
||||||
CURLcode(*)(CURL *, CURLINFO info, ...));
|
CURLcode(*)(CURL *, CURLINFO info, ...));
|
||||||
|
|
||||||
|
SET_AND_CHECK_FUNCTION_POINTER(easy_reset_,
|
||||||
|
"curl_easy_reset",
|
||||||
|
void(*)(CURL*));
|
||||||
|
|
||||||
SET_AND_CHECK_FUNCTION_POINTER(slist_free_all_,
|
SET_AND_CHECK_FUNCTION_POINTER(slist_free_all_,
|
||||||
"curl_slist_free_all",
|
"curl_slist_free_all",
|
||||||
void(*)(curl_slist*));
|
void(*)(curl_slist*));
|
||||||
|
|
@ -238,4 +254,73 @@ bool LibcurlWrapper::SetFunctionPointers() {
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
bool LibcurlWrapper::SendRequestInner(const string& url,
|
||||||
|
long* http_status_code,
|
||||||
|
string* http_header_data,
|
||||||
|
string* http_response_data) {
|
||||||
|
string url_copy(url);
|
||||||
|
(*easy_setopt_)(curl_, CURLOPT_URL, url_copy.c_str());
|
||||||
|
|
||||||
|
// Disable 100-continue header.
|
||||||
|
char buf[] = "Expect:";
|
||||||
|
headerlist_ = (*slist_append_)(headerlist_, buf);
|
||||||
|
(*easy_setopt_)(curl_, CURLOPT_HTTPHEADER, headerlist_);
|
||||||
|
|
||||||
|
if (http_response_data != nullptr) {
|
||||||
|
http_response_data->clear();
|
||||||
|
(*easy_setopt_)(curl_, CURLOPT_WRITEFUNCTION, WriteCallback);
|
||||||
|
(*easy_setopt_)(curl_, CURLOPT_WRITEDATA,
|
||||||
|
reinterpret_cast<void*>(http_response_data));
|
||||||
|
}
|
||||||
|
if (http_header_data != nullptr) {
|
||||||
|
http_header_data->clear();
|
||||||
|
(*easy_setopt_)(curl_, CURLOPT_HEADERFUNCTION, WriteCallback);
|
||||||
|
(*easy_setopt_)(curl_, CURLOPT_HEADERDATA,
|
||||||
|
reinterpret_cast<void*>(http_header_data));
|
||||||
|
}
|
||||||
|
CURLcode err_code = CURLE_OK;
|
||||||
|
err_code = (*easy_perform_)(curl_);
|
||||||
|
easy_strerror_ = reinterpret_cast<const char* (*)(CURLcode)>
|
||||||
|
(dlsym(curl_lib_, "curl_easy_strerror"));
|
||||||
|
|
||||||
|
if (http_status_code != nullptr) {
|
||||||
|
(*easy_getinfo_)(curl_, CURLINFO_RESPONSE_CODE, http_status_code);
|
||||||
|
}
|
||||||
|
|
||||||
|
#ifndef NDEBUG
|
||||||
|
if (err_code != CURLE_OK)
|
||||||
|
fprintf(stderr, "Failed to send http request to %s, error: %s\n",
|
||||||
|
url.c_str(),
|
||||||
|
(*easy_strerror_)(err_code));
|
||||||
|
#endif
|
||||||
|
|
||||||
|
Reset();
|
||||||
|
|
||||||
|
return err_code == CURLE_OK;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void LibcurlWrapper::Reset() {
|
||||||
|
if (headerlist_ != nullptr) {
|
||||||
|
(*slist_free_all_)(headerlist_);
|
||||||
|
headerlist_ = nullptr;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (formpost_ != nullptr) {
|
||||||
|
(*formfree_)(formpost_);
|
||||||
|
formpost_ = nullptr;
|
||||||
|
}
|
||||||
|
|
||||||
|
(*easy_reset_)(curl_);
|
||||||
|
}
|
||||||
|
|
||||||
|
bool LibcurlWrapper::CheckInit() {
|
||||||
|
if (!init_ok_) {
|
||||||
|
std::cout << "LibcurlWrapper: You must call Init(), and have it return "
|
||||||
|
"'true' before invoking any other methods.\n";
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
} // namespace google_breakpad
|
||||||
|
|
|
||||||
|
|
@ -51,14 +51,39 @@ class LibcurlWrapper {
|
||||||
const string& basename);
|
const string& basename);
|
||||||
virtual bool SendRequest(const string& url,
|
virtual bool SendRequest(const string& url,
|
||||||
const std::map<string, string>& parameters,
|
const std::map<string, string>& parameters,
|
||||||
int* http_status_code,
|
long* http_status_code,
|
||||||
string* http_header_data,
|
string* http_header_data,
|
||||||
string* http_response_data);
|
string* http_response_data);
|
||||||
|
bool SendGetRequest(const string& url,
|
||||||
|
long* http_status_code,
|
||||||
|
string* http_header_data,
|
||||||
|
string* http_response_data);
|
||||||
|
bool SendPutRequest(const string& url,
|
||||||
|
const string& path,
|
||||||
|
long* http_status_code,
|
||||||
|
string* http_header_data,
|
||||||
|
string* http_response_data);
|
||||||
|
bool SendSimplePostRequest(const string& url,
|
||||||
|
const string& body,
|
||||||
|
const string& content_type,
|
||||||
|
long* http_status_code,
|
||||||
|
string* http_header_data,
|
||||||
|
string* http_response_data);
|
||||||
|
|
||||||
private:
|
private:
|
||||||
// This function initializes class state corresponding to function
|
// This function initializes class state corresponding to function
|
||||||
// pointers into the CURL library.
|
// pointers into the CURL library.
|
||||||
bool SetFunctionPointers();
|
bool SetFunctionPointers();
|
||||||
|
|
||||||
|
bool SendRequestInner(const string& url,
|
||||||
|
long* http_status_code,
|
||||||
|
string* http_header_data,
|
||||||
|
string* http_response_data);
|
||||||
|
|
||||||
|
void Reset();
|
||||||
|
|
||||||
|
bool CheckInit();
|
||||||
|
|
||||||
bool init_ok_; // Whether init succeeded
|
bool init_ok_; // Whether init succeeded
|
||||||
void* curl_lib_; // Pointer to result of dlopen() on
|
void* curl_lib_; // Pointer to result of dlopen() on
|
||||||
// curl library
|
// curl library
|
||||||
|
|
@ -85,6 +110,7 @@ class LibcurlWrapper {
|
||||||
const char* (*easy_strerror_)(CURLcode);
|
const char* (*easy_strerror_)(CURLcode);
|
||||||
void (*easy_cleanup_)(CURL *);
|
void (*easy_cleanup_)(CURL *);
|
||||||
CURLcode (*easy_getinfo_)(CURL *, CURLINFO info, ...);
|
CURLcode (*easy_getinfo_)(CURL *, CURLINFO info, ...);
|
||||||
|
void (*easy_reset_)(CURL*);
|
||||||
void (*formfree_)(struct curl_httppost *);
|
void (*formfree_)(struct curl_httppost *);
|
||||||
|
|
||||||
};
|
};
|
||||||
|
|
|
||||||
193
src/common/linux/symbol_collector_client.cc
Normal file
193
src/common/linux/symbol_collector_client.cc
Normal file
|
|
@ -0,0 +1,193 @@
|
||||||
|
// Copyright (c) 2019 Google Inc.
|
||||||
|
// All rights reserved.
|
||||||
|
//
|
||||||
|
// Redistribution and use in source and binary forms, with or without
|
||||||
|
// modification, are permitted provided that the following conditions are
|
||||||
|
// met:
|
||||||
|
//
|
||||||
|
// * Redistributions of source code must retain the above copyright
|
||||||
|
// notice, this list of conditions and the following disclaimer.
|
||||||
|
// * Redistributions in binary form must reproduce the above
|
||||||
|
// copyright notice, this list of conditions and the following disclaimer
|
||||||
|
// in the documentation and/or other materials provided with the
|
||||||
|
// distribution.
|
||||||
|
// * Neither the name of Google Inc. nor the names of its
|
||||||
|
// contributors may be used to endorse or promote products derived from
|
||||||
|
// this software without specific prior written permission.
|
||||||
|
//
|
||||||
|
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||||
|
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||||
|
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||||
|
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||||
|
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||||
|
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||||
|
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||||
|
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||||
|
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||||
|
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||||
|
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||||
|
|
||||||
|
#include "common/linux/symbol_collector_client.h"
|
||||||
|
|
||||||
|
#include <stdio.h>
|
||||||
|
|
||||||
|
#include <iostream>
|
||||||
|
#include <regex>
|
||||||
|
|
||||||
|
#include "common/linux/libcurl_wrapper.h"
|
||||||
|
|
||||||
|
namespace google_breakpad {
|
||||||
|
namespace sym_upload {
|
||||||
|
|
||||||
|
// static
|
||||||
|
bool SymbolCollectorClient::CreateUploadUrl(
|
||||||
|
LibcurlWrapper* libcurl_wrapper,
|
||||||
|
const string& api_url,
|
||||||
|
const string& api_key,
|
||||||
|
UploadUrlResponse* uploadUrlResponse) {
|
||||||
|
string header, response;
|
||||||
|
long response_code;
|
||||||
|
|
||||||
|
string url = api_url + "/v1/uploads:create";
|
||||||
|
if (!api_key.empty()) {
|
||||||
|
url += "?key=" + api_key;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!libcurl_wrapper->SendSimplePostRequest(url,
|
||||||
|
/*body=*/"",
|
||||||
|
/*content_type=*/"",
|
||||||
|
&response_code,
|
||||||
|
&header,
|
||||||
|
&response)) {
|
||||||
|
printf("Failed to create upload url.\n");
|
||||||
|
printf("Response code: %ld\n", response_code);
|
||||||
|
printf("Response:\n");
|
||||||
|
printf("%s\n", response.c_str());
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Note camel-case rather than underscores.
|
||||||
|
std::regex upload_url_regex("\"uploadUrl\": \"([^\"]+)\"");
|
||||||
|
std::regex upload_key_regex("\"uploadKey\": \"([^\"]+)\"");
|
||||||
|
|
||||||
|
std::smatch upload_url_match;
|
||||||
|
if (!std::regex_search(response, upload_url_match, upload_url_regex) ||
|
||||||
|
upload_url_match.size() != 2) {
|
||||||
|
printf("Failed to parse create url response.");
|
||||||
|
printf("Response:\n");
|
||||||
|
printf("%s\n", response.c_str());
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
string upload_url = upload_url_match[1].str();
|
||||||
|
|
||||||
|
std::smatch upload_key_match;
|
||||||
|
if (!std::regex_search(response, upload_key_match, upload_key_regex) ||
|
||||||
|
upload_key_match.size() != 2) {
|
||||||
|
printf("Failed to parse create url response.");
|
||||||
|
printf("Response:\n");
|
||||||
|
printf("%s\n", response.c_str());
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
string upload_key = upload_key_match[1].str();
|
||||||
|
|
||||||
|
uploadUrlResponse->upload_url = upload_url;
|
||||||
|
uploadUrlResponse->upload_key = upload_key;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
// static
|
||||||
|
CompleteUploadResult SymbolCollectorClient::CompleteUpload(
|
||||||
|
LibcurlWrapper* libcurl_wrapper,
|
||||||
|
const string& api_url,
|
||||||
|
const string& api_key,
|
||||||
|
const string& upload_key,
|
||||||
|
const string& debug_file,
|
||||||
|
const string& debug_id) {
|
||||||
|
string header, response;
|
||||||
|
long response_code;
|
||||||
|
|
||||||
|
string url = api_url + "/v1/uploads/" + upload_key + ":complete";
|
||||||
|
if (!api_key.empty()) {
|
||||||
|
url += "?key=" + api_key;
|
||||||
|
}
|
||||||
|
string body =
|
||||||
|
"{ symbol_id: {"
|
||||||
|
"debug_file: \"" + debug_file + "\", "
|
||||||
|
"debug_id: \"" + debug_id + "\" } }";
|
||||||
|
|
||||||
|
if (!libcurl_wrapper->SendSimplePostRequest(url,
|
||||||
|
body,
|
||||||
|
"application/son",
|
||||||
|
&response_code,
|
||||||
|
&header,
|
||||||
|
&response)) {
|
||||||
|
printf("Failed to complete upload.\n");
|
||||||
|
printf("Response code: %ld\n", response_code);
|
||||||
|
printf("Response:\n");
|
||||||
|
printf("%s\n", response.c_str());
|
||||||
|
return CompleteUploadResult::Error;
|
||||||
|
}
|
||||||
|
|
||||||
|
std::regex result_regex("\"result\": \"([^\"]+)\"");
|
||||||
|
std::smatch result_match;
|
||||||
|
if (!std::regex_search(response, result_match, result_regex) ||
|
||||||
|
result_match.size() != 2) {
|
||||||
|
printf("Failed to parse complete upload response.");
|
||||||
|
printf("Response:\n");
|
||||||
|
printf("%s\n", response.c_str());
|
||||||
|
return CompleteUploadResult::Error;
|
||||||
|
}
|
||||||
|
string result = result_match[1].str();
|
||||||
|
|
||||||
|
if (result.compare("DUPLICATE_DATA") == 0) {
|
||||||
|
return CompleteUploadResult::DuplicateData;
|
||||||
|
}
|
||||||
|
|
||||||
|
return CompleteUploadResult::Ok;
|
||||||
|
}
|
||||||
|
|
||||||
|
// static
|
||||||
|
SymbolStatus SymbolCollectorClient::CheckSymbolStatus(
|
||||||
|
LibcurlWrapper* libcurl_wrapper,
|
||||||
|
const string& api_url,
|
||||||
|
const string& api_key,
|
||||||
|
const string& debug_file,
|
||||||
|
const string& debug_id) {
|
||||||
|
string header, response;
|
||||||
|
long response_code;
|
||||||
|
string url = api_url +
|
||||||
|
"/v1/symbols/" + debug_file + "/" + debug_id + ":checkStatus";
|
||||||
|
if (!api_key.empty()) {
|
||||||
|
url += "?key=" + api_key;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!libcurl_wrapper->SendGetRequest(
|
||||||
|
url,
|
||||||
|
&response_code,
|
||||||
|
&header,
|
||||||
|
&response)) {
|
||||||
|
printf("Failed to check symbol status, error message.\n");
|
||||||
|
printf("Response code: %ld\n", response_code);
|
||||||
|
printf("Response:\n");
|
||||||
|
printf("%s\n", response.c_str());
|
||||||
|
return SymbolStatus::Unknown;
|
||||||
|
}
|
||||||
|
|
||||||
|
std::regex status_regex("\"status\": \"([^\"]+)\"");
|
||||||
|
std::smatch status_match;
|
||||||
|
if (!std::regex_search(response, status_match, status_regex) ||
|
||||||
|
status_match.size() != 2) {
|
||||||
|
printf("Failed to parse check symbol status response.");
|
||||||
|
printf("Response:\n");
|
||||||
|
printf("%s\n", response.c_str());
|
||||||
|
return SymbolStatus::Unknown;
|
||||||
|
}
|
||||||
|
string status = status_match[1].str();
|
||||||
|
|
||||||
|
return (status.compare("FOUND") == 0) ?
|
||||||
|
SymbolStatus::Found :
|
||||||
|
SymbolStatus::Missing;
|
||||||
|
}
|
||||||
|
|
||||||
|
} // namespace sym_upload
|
||||||
|
} // namespace google_breakpad
|
||||||
87
src/common/linux/symbol_collector_client.h
Normal file
87
src/common/linux/symbol_collector_client.h
Normal file
|
|
@ -0,0 +1,87 @@
|
||||||
|
// Copyright (c) 2019, Google Inc.
|
||||||
|
// All rights reserved.
|
||||||
|
//
|
||||||
|
// Redistribution and use in source and binary forms, with or without
|
||||||
|
// modification, are permitted provided that the following conditions are
|
||||||
|
// met:
|
||||||
|
//
|
||||||
|
// * Redistributions of source code must retain the above copyright
|
||||||
|
// notice, this list of conditions and the following disclaimer.
|
||||||
|
// * Redistributions in binary form must reproduce the above
|
||||||
|
// copyright notice, this list of conditions and the following disclaimer
|
||||||
|
// in the documentation and/or other materials provided with the
|
||||||
|
// distribution.
|
||||||
|
// * Neither the name of Google Inc. nor the names of its
|
||||||
|
// contributors may be used to endorse or promote products derived from
|
||||||
|
// this software without specific prior written permission.
|
||||||
|
//
|
||||||
|
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||||
|
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||||
|
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||||
|
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||||
|
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||||
|
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||||
|
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||||
|
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||||
|
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||||
|
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||||
|
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||||
|
|
||||||
|
#ifndef COMMON_LINUX_SYMBOL_COLLECTOR_CLIENT_H_
|
||||||
|
#define COMMON_LINUX_SYMBOL_COLLECTOR_CLIENT_H_
|
||||||
|
|
||||||
|
#include <string>
|
||||||
|
|
||||||
|
#include "common/linux/libcurl_wrapper.h"
|
||||||
|
#include "common/using_std_string.h"
|
||||||
|
|
||||||
|
namespace google_breakpad {
|
||||||
|
namespace sym_upload {
|
||||||
|
|
||||||
|
struct UploadUrlResponse {
|
||||||
|
string upload_url;
|
||||||
|
string upload_key;
|
||||||
|
};
|
||||||
|
|
||||||
|
enum SymbolStatus {
|
||||||
|
Found,
|
||||||
|
Missing,
|
||||||
|
Unknown
|
||||||
|
};
|
||||||
|
|
||||||
|
enum CompleteUploadResult {
|
||||||
|
Ok,
|
||||||
|
DuplicateData,
|
||||||
|
Error
|
||||||
|
};
|
||||||
|
|
||||||
|
// Helper class to communicate with a sym-upload-v2 service over HTTP/REST,
|
||||||
|
// via libcurl.
|
||||||
|
class SymbolCollectorClient {
|
||||||
|
public:
|
||||||
|
static bool CreateUploadUrl(
|
||||||
|
LibcurlWrapper* libcurl_wrapper,
|
||||||
|
const string& api_url,
|
||||||
|
const string& api_key,
|
||||||
|
UploadUrlResponse* uploadUrlResponse);
|
||||||
|
|
||||||
|
static CompleteUploadResult CompleteUpload(
|
||||||
|
LibcurlWrapper* libcurl_wrapper,
|
||||||
|
const string& api_url,
|
||||||
|
const string& api_key,
|
||||||
|
const string& upload_key,
|
||||||
|
const string& debug_file,
|
||||||
|
const string& debug_id);
|
||||||
|
|
||||||
|
static SymbolStatus CheckSymbolStatus(
|
||||||
|
LibcurlWrapper* libcurl_wrapper,
|
||||||
|
const string& api_url,
|
||||||
|
const string& api_key,
|
||||||
|
const string& debug_file,
|
||||||
|
const string& debug_id);
|
||||||
|
};
|
||||||
|
|
||||||
|
} // namespace sym_upload
|
||||||
|
} // namespace google_breakpad
|
||||||
|
|
||||||
|
#endif // COMMON_LINUX_SYMBOL_COLLECTOR_CLIENT_H_
|
||||||
|
|
@ -30,15 +30,19 @@
|
||||||
// symbol_upload.cc: implemented google_breakpad::sym_upload::Start, a helper
|
// symbol_upload.cc: implemented google_breakpad::sym_upload::Start, a helper
|
||||||
// function for linux symbol upload tool.
|
// function for linux symbol upload tool.
|
||||||
|
|
||||||
#include "common/linux/http_upload.h"
|
|
||||||
#include "common/linux/symbol_upload.h"
|
#include "common/linux/symbol_upload.h"
|
||||||
|
|
||||||
#include <assert.h>
|
#include <assert.h>
|
||||||
#include <stdio.h>
|
#include <stdio.h>
|
||||||
|
|
||||||
#include <functional>
|
#include <functional>
|
||||||
|
#include <iostream>
|
||||||
#include <vector>
|
#include <vector>
|
||||||
|
|
||||||
|
#include "common/linux/http_upload.h"
|
||||||
|
#include "common/linux/libcurl_wrapper.h"
|
||||||
|
#include "common/linux/symbol_collector_client.h"
|
||||||
|
|
||||||
namespace google_breakpad {
|
namespace google_breakpad {
|
||||||
namespace sym_upload {
|
namespace sym_upload {
|
||||||
|
|
||||||
|
|
@ -95,21 +99,19 @@ string CompactIdentifier(const string &uuid) {
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
//=============================================================================
|
// |options| describes the current sym_upload options.
|
||||||
void Start(Options *options) {
|
// |module_parts| contains the strings parsed from the MODULE entry of the
|
||||||
|
// Breakpad symbol file being uploaded.
|
||||||
|
// |compacted_id| is the debug_id from the MODULE entry of the Breakpad symbol
|
||||||
|
// file being uploaded, with all hyphens removed.
|
||||||
|
bool SymUploadV1Start(
|
||||||
|
const Options& options,
|
||||||
|
std::vector<string> module_parts,
|
||||||
|
const string& compacted_id) {
|
||||||
std::map<string, string> parameters;
|
std::map<string, string> parameters;
|
||||||
options->success = false;
|
|
||||||
std::vector<string> module_parts;
|
|
||||||
if (!ModuleDataForSymbolFile(options->symbolsPath, &module_parts)) {
|
|
||||||
fprintf(stderr, "Failed to parse symbol file!\n");
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
string compacted_id = CompactIdentifier(module_parts[3]);
|
|
||||||
|
|
||||||
// Add parameters
|
// Add parameters
|
||||||
if (!options->version.empty())
|
if (!options.version.empty())
|
||||||
parameters["version"] = options->version;
|
parameters["version"] = options.version;
|
||||||
|
|
||||||
// MODULE <os> <cpu> <uuid> <module-name>
|
// MODULE <os> <cpu> <uuid> <module-name>
|
||||||
// 0 1 2 3 4
|
// 0 1 2 3 4
|
||||||
|
|
@ -120,16 +122,16 @@ void Start(Options *options) {
|
||||||
parameters["debug_identifier"] = compacted_id;
|
parameters["debug_identifier"] = compacted_id;
|
||||||
|
|
||||||
std::map<string, string> files;
|
std::map<string, string> files;
|
||||||
files["symbol_file"] = options->symbolsPath;
|
files["symbol_file"] = options.symbolsPath;
|
||||||
|
|
||||||
string response, error;
|
string response, error;
|
||||||
long response_code;
|
long response_code;
|
||||||
bool success = HTTPUpload::SendRequest(options->uploadURLStr,
|
bool success = HTTPUpload::SendRequest(options.uploadURLStr,
|
||||||
parameters,
|
parameters,
|
||||||
files,
|
files,
|
||||||
options->proxy,
|
options.proxy,
|
||||||
options->proxy_user_pwd,
|
options.proxy_user_pwd,
|
||||||
"",
|
/*ca_certificate_file=*/"",
|
||||||
&response,
|
&response,
|
||||||
&response_code,
|
&response_code,
|
||||||
&error);
|
&error);
|
||||||
|
|
@ -148,7 +150,116 @@ void Start(Options *options) {
|
||||||
} else {
|
} else {
|
||||||
printf("Successfully sent the symbol file.\n");
|
printf("Successfully sent the symbol file.\n");
|
||||||
}
|
}
|
||||||
options->success = success;
|
|
||||||
|
return success;
|
||||||
|
}
|
||||||
|
|
||||||
|
// |options| describes the current sym_upload options.
|
||||||
|
// |module_parts| contains the strings parsed from the MODULE entry of the
|
||||||
|
// Breakpad symbol file being uploaded.
|
||||||
|
// |compacted_id| is the debug_id from the MODULE entry of the Breakpad symbol
|
||||||
|
// file being uploaded, with all hyphens removed.
|
||||||
|
bool SymUploadV2Start(
|
||||||
|
const Options& options,
|
||||||
|
std::vector<string> module_parts,
|
||||||
|
const string& compacted_id) {
|
||||||
|
string debug_file = module_parts[4];
|
||||||
|
string debug_id = compacted_id;
|
||||||
|
|
||||||
|
google_breakpad::LibcurlWrapper libcurl_wrapper;
|
||||||
|
if (!libcurl_wrapper.Init()) {
|
||||||
|
printf("Failed to init google_breakpad::LibcurlWrapper.\n");
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!options.force) {
|
||||||
|
SymbolStatus symbolStatus = SymbolCollectorClient::CheckSymbolStatus(
|
||||||
|
&libcurl_wrapper,
|
||||||
|
options.uploadURLStr,
|
||||||
|
options.api_key,
|
||||||
|
debug_file,
|
||||||
|
debug_id);
|
||||||
|
if (symbolStatus == SymbolStatus::Found) {
|
||||||
|
printf("Symbol file already exists, upload aborted."
|
||||||
|
" Use \"-f\" to overwrite.\n");
|
||||||
|
return true;
|
||||||
|
} else if (symbolStatus == SymbolStatus::Unknown) {
|
||||||
|
printf("Failed to check for existing symbol.\n");
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
UploadUrlResponse uploadUrlResponse;
|
||||||
|
if (!SymbolCollectorClient::CreateUploadUrl(
|
||||||
|
&libcurl_wrapper,
|
||||||
|
options.uploadURLStr,
|
||||||
|
options.api_key,
|
||||||
|
&uploadUrlResponse)) {
|
||||||
|
printf("Failed to create upload URL.\n");
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
string signed_url = uploadUrlResponse.upload_url;
|
||||||
|
string upload_key = uploadUrlResponse.upload_key;
|
||||||
|
string header;
|
||||||
|
string response;
|
||||||
|
long response_code;
|
||||||
|
|
||||||
|
if (!libcurl_wrapper.SendPutRequest(signed_url,
|
||||||
|
options.symbolsPath,
|
||||||
|
&response_code,
|
||||||
|
&header,
|
||||||
|
&response)) {
|
||||||
|
printf("Failed to send symbol file.\n");
|
||||||
|
printf("Response code: %ld\n", response_code);
|
||||||
|
printf("Response:\n");
|
||||||
|
printf("%s\n", response.c_str());
|
||||||
|
return false;
|
||||||
|
} else if (response_code == 0) {
|
||||||
|
printf("Failed to send symbol file: No response code\n");
|
||||||
|
return false;
|
||||||
|
} else if (response_code != 200) {
|
||||||
|
printf("Failed to send symbol file: Response code %ld\n", response_code);
|
||||||
|
printf("Response:\n");
|
||||||
|
printf("%s\n", response.c_str());
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
CompleteUploadResult completeUploadResult =
|
||||||
|
SymbolCollectorClient::CompleteUpload(&libcurl_wrapper,
|
||||||
|
options.uploadURLStr,
|
||||||
|
options.api_key,
|
||||||
|
upload_key,
|
||||||
|
debug_file,
|
||||||
|
debug_id);
|
||||||
|
if (completeUploadResult == CompleteUploadResult::Error) {
|
||||||
|
printf("Failed to complete upload.\n");
|
||||||
|
return false;
|
||||||
|
} else if (completeUploadResult == CompleteUploadResult::DuplicateData) {
|
||||||
|
printf("Uploaded file checksum matched existing file checksum,"
|
||||||
|
" no change necessary.\n");
|
||||||
|
} else {
|
||||||
|
printf("Successfully sent the symbol file.\n");
|
||||||
|
}
|
||||||
|
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
//=============================================================================
|
||||||
|
void Start(Options* options) {
|
||||||
|
std::vector<string> module_parts;
|
||||||
|
if (!ModuleDataForSymbolFile(options->symbolsPath, &module_parts)) {
|
||||||
|
fprintf(stderr, "Failed to parse symbol file!\n");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const string compacted_id = CompactIdentifier(module_parts[3]);
|
||||||
|
|
||||||
|
if (options->upload_protocol == UploadProtocol::SYM_UPLOAD_V2) {
|
||||||
|
options->success = SymUploadV2Start(*options, module_parts, compacted_id);
|
||||||
|
} else {
|
||||||
|
options->success = SymUploadV1Start(*options, module_parts, compacted_id);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
} // namespace sym_upload
|
} // namespace sym_upload
|
||||||
|
|
|
||||||
|
|
@ -41,14 +41,24 @@
|
||||||
namespace google_breakpad {
|
namespace google_breakpad {
|
||||||
namespace sym_upload {
|
namespace sym_upload {
|
||||||
|
|
||||||
typedef struct {
|
enum class UploadProtocol {
|
||||||
|
SYM_UPLOAD_V1,
|
||||||
|
SYM_UPLOAD_V2,
|
||||||
|
};
|
||||||
|
|
||||||
|
struct Options {
|
||||||
|
Options() : upload_protocol(UploadProtocol::SYM_UPLOAD_V1), force(false) {}
|
||||||
|
|
||||||
string symbolsPath;
|
string symbolsPath;
|
||||||
string uploadURLStr;
|
string uploadURLStr;
|
||||||
string proxy;
|
string proxy;
|
||||||
string proxy_user_pwd;
|
string proxy_user_pwd;
|
||||||
string version;
|
string version;
|
||||||
bool success;
|
bool success;
|
||||||
} Options;
|
UploadProtocol upload_protocol;
|
||||||
|
bool force;
|
||||||
|
string api_key;
|
||||||
|
};
|
||||||
|
|
||||||
// Starts upload to symbol server with options.
|
// Starts upload to symbol server with options.
|
||||||
void Start(Options* options);
|
void Start(Options* options);
|
||||||
|
|
|
||||||
|
|
@ -41,25 +41,43 @@
|
||||||
|
|
||||||
#include <stdio.h>
|
#include <stdio.h>
|
||||||
#include <stdlib.h>
|
#include <stdlib.h>
|
||||||
|
#include <string.h>
|
||||||
#include <unistd.h>
|
#include <unistd.h>
|
||||||
|
|
||||||
#include "common/linux/symbol_upload.h"
|
#include "common/linux/symbol_upload.h"
|
||||||
|
|
||||||
|
using google_breakpad::sym_upload::UploadProtocol;
|
||||||
using google_breakpad::sym_upload::Options;
|
using google_breakpad::sym_upload::Options;
|
||||||
|
|
||||||
//=============================================================================
|
//=============================================================================
|
||||||
static void
|
static void
|
||||||
Usage(int argc, const char *argv[]) {
|
Usage(int argc, const char *argv[]) {
|
||||||
fprintf(stderr, "Submit symbol information.\n");
|
fprintf(stderr, "Submit symbol information.\n");
|
||||||
fprintf(stderr, "Usage: %s [options...] <symbols> <upload-URL>\n", argv[0]);
|
fprintf(stderr, "Usage: %s [options...] <symbol-file> <upload-URL>\n",
|
||||||
|
argv[0]);
|
||||||
fprintf(stderr, "Options:\n");
|
fprintf(stderr, "Options:\n");
|
||||||
fprintf(stderr, "<symbols> should be created by using the dump_syms tool.\n");
|
fprintf(stderr, "<symbol-file> should be created by using the dump_syms"
|
||||||
|
"tool.\n");
|
||||||
fprintf(stderr, "<upload-URL> is the destination for the upload\n");
|
fprintf(stderr, "<upload-URL> is the destination for the upload\n");
|
||||||
|
fprintf(stderr, "-p:\t <protocol> One of ['sym-upload-v1',"
|
||||||
|
" 'sym-upload-v2'], defaults to 'sym-upload-v1'.\n");
|
||||||
|
fprintf(stderr, "-k:\t <API-key> A secret used to authenticate with the"
|
||||||
|
" API [Only supported when using 'sym-upload-v2' protocol].\n");
|
||||||
|
fprintf(stderr, "-f:\t Force symbol upload if already exists [Only"
|
||||||
|
" supported when using 'sym-upload-v2' protocol].\n");
|
||||||
fprintf(stderr, "-v:\t Version information (e.g., 1.2.3.4)\n");
|
fprintf(stderr, "-v:\t Version information (e.g., 1.2.3.4)\n");
|
||||||
fprintf(stderr, "-x:\t <host[:port]> Use HTTP proxy on given port\n");
|
fprintf(stderr, "-x:\t <host[:port]> Use HTTP proxy on given port\n");
|
||||||
fprintf(stderr, "-u:\t <user[:password]> Set proxy user and password\n");
|
fprintf(stderr, "-u:\t <user[:password]> Set proxy user and password\n");
|
||||||
fprintf(stderr, "-h:\t Usage\n");
|
fprintf(stderr, "-h:\t Usage\n");
|
||||||
fprintf(stderr, "-?:\t Usage\n");
|
fprintf(stderr, "-?:\t Usage\n");
|
||||||
|
fprintf(stderr, "\n");
|
||||||
|
fprintf(stderr, "Examples:\n");
|
||||||
|
fprintf(stderr, " With 'sym-upload-v1':\n");
|
||||||
|
fprintf(stderr, " %s path/to/symbol_file http://myuploadserver\n",
|
||||||
|
argv[0]);
|
||||||
|
fprintf(stderr, " With 'sym-upload-v2':\n");
|
||||||
|
fprintf(stderr, " %s -p sym-upload-v2 -k mysecret123! "
|
||||||
|
"path/to/symbol_file http://myuploadserver\n", argv[0]);
|
||||||
}
|
}
|
||||||
|
|
||||||
//=============================================================================
|
//=============================================================================
|
||||||
|
|
@ -68,7 +86,7 @@ SetupOptions(int argc, const char *argv[], Options *options) {
|
||||||
extern int optind;
|
extern int optind;
|
||||||
int ch;
|
int ch;
|
||||||
|
|
||||||
while ((ch = getopt(argc, (char * const *)argv, "u:v:x:h?")) != -1) {
|
while ((ch = getopt(argc, (char * const *)argv, "u:v:x:p:k:hf?")) != -1) {
|
||||||
switch (ch) {
|
switch (ch) {
|
||||||
case 'h':
|
case 'h':
|
||||||
case '?':
|
case '?':
|
||||||
|
|
@ -84,6 +102,23 @@ SetupOptions(int argc, const char *argv[], Options *options) {
|
||||||
case 'x':
|
case 'x':
|
||||||
options->proxy = optarg;
|
options->proxy = optarg;
|
||||||
break;
|
break;
|
||||||
|
case 'p':
|
||||||
|
if (strcmp(optarg, "sym-upload-v2") == 0) {
|
||||||
|
options->upload_protocol = UploadProtocol::SYM_UPLOAD_V2;
|
||||||
|
} else if (strcmp(optarg, "sym-upload-v1") == 0) {
|
||||||
|
options->upload_protocol = UploadProtocol::SYM_UPLOAD_V1;
|
||||||
|
} else {
|
||||||
|
fprintf(stderr, "Invalid protocol '%c'\n", optarg);
|
||||||
|
Usage(argc, argv);
|
||||||
|
exit(1);
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
case 'k':
|
||||||
|
options->api_key = optarg;
|
||||||
|
break;
|
||||||
|
case 'f':
|
||||||
|
options->force = true;
|
||||||
|
break;
|
||||||
|
|
||||||
default:
|
default:
|
||||||
fprintf(stderr, "Invalid option '%c'\n", ch);
|
fprintf(stderr, "Invalid option '%c'\n", ch);
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue