Add optional new symbol upload API to sym_upload.

Change-Id: I6a49e9f4a699fa6f5f8e9f0fc86afb4cb342a442
Reviewed-on: https://chromium-review.googlesource.com/c/breakpad/breakpad/+/1422400
Reviewed-by: Mark Mentovai <mark@chromium.org>
Reviewed-by: Ivan Penkov <ivanpe@chromium.org>
Reviewed-by: Mike Frysinger <vapier@chromium.org>

Makefile.am

@@ -615,6 +615,10 @@ src_tools_linux_symupload_minidump_upload_LDADD = -ldl
 src_tools_linux_symupload_sym_upload_SOURCES = \
 	src/common/linux/http_upload.cc \
 	src/common/linux/http_upload.h \
+	src/common/linux/libcurl_wrapper.cc \
+	src/common/linux/libcurl_wrapper.h \
+	src/common/linux/symbol_collector_client.cc \
+	src/common/linux/symbol_collector_client.h \
 	src/common/linux/symbol_upload.cc \
 	src/common/linux/symbol_upload.h \
 	src/tools/linux/symupload/sym_upload.cc

Makefile.in

@@ -1495,10 +1495,16 @@ src_tools_linux_symupload_minidump_upload_OBJECTS =  \
 src_tools_linux_symupload_minidump_upload_DEPENDENCIES =
 am__src_tools_linux_symupload_sym_upload_SOURCES_DIST =  \
 	src/common/linux/http_upload.cc src/common/linux/http_upload.h \
+	src/common/linux/libcurl_wrapper.cc \
+	src/common/linux/libcurl_wrapper.h \
+	src/common/linux/symbol_collector_client.cc \
+	src/common/linux/symbol_collector_client.h \
 	src/common/linux/symbol_upload.cc \
 	src/common/linux/symbol_upload.h \
 	src/tools/linux/symupload/sym_upload.cc
 @DISABLE_TOOLS_FALSE@@LINUX_HOST_TRUE@am_src_tools_linux_symupload_sym_upload_OBJECTS = src/common/linux/http_upload.$(OBJEXT) \
+@DISABLE_TOOLS_FALSE@@LINUX_HOST_TRUE@	src/common/linux/libcurl_wrapper.$(OBJEXT) \
+@DISABLE_TOOLS_FALSE@@LINUX_HOST_TRUE@	src/common/linux/symbol_collector_client.$(OBJEXT) \
 @DISABLE_TOOLS_FALSE@@LINUX_HOST_TRUE@	src/common/linux/symbol_upload.$(OBJEXT) \
 @DISABLE_TOOLS_FALSE@@LINUX_HOST_TRUE@	src/tools/linux/symupload/sym_upload.$(OBJEXT)
 src_tools_linux_symupload_sym_upload_OBJECTS =  \
@@ -2477,6 +2483,10 @@ TESTS = $(check_PROGRAMS) $(check_SCRIPTS)
 @DISABLE_TOOLS_FALSE@@LINUX_HOST_TRUE@src_tools_linux_symupload_sym_upload_SOURCES = \
 @DISABLE_TOOLS_FALSE@@LINUX_HOST_TRUE@	src/common/linux/http_upload.cc \
 @DISABLE_TOOLS_FALSE@@LINUX_HOST_TRUE@	src/common/linux/http_upload.h \
+@DISABLE_TOOLS_FALSE@@LINUX_HOST_TRUE@	src/common/linux/libcurl_wrapper.cc \
+@DISABLE_TOOLS_FALSE@@LINUX_HOST_TRUE@	src/common/linux/libcurl_wrapper.h \
+@DISABLE_TOOLS_FALSE@@LINUX_HOST_TRUE@	src/common/linux/symbol_collector_client.cc \
+@DISABLE_TOOLS_FALSE@@LINUX_HOST_TRUE@	src/common/linux/symbol_collector_client.h \
 @DISABLE_TOOLS_FALSE@@LINUX_HOST_TRUE@	src/common/linux/symbol_upload.cc \
 @DISABLE_TOOLS_FALSE@@LINUX_HOST_TRUE@	src/common/linux/symbol_upload.h \
 @DISABLE_TOOLS_FALSE@@LINUX_HOST_TRUE@	src/tools/linux/symupload/sym_upload.cc
@@ -4726,6 +4736,9 @@ src/tools/linux/symupload/minidump_upload.$(OBJEXT):  \
 src/tools/linux/symupload/minidump_upload$(EXEEXT): $(src_tools_linux_symupload_minidump_upload_OBJECTS) $(src_tools_linux_symupload_minidump_upload_DEPENDENCIES) $(EXTRA_src_tools_linux_symupload_minidump_upload_DEPENDENCIES) src/tools/linux/symupload/$(am__dirstamp)
 	@rm -f src/tools/linux/symupload/minidump_upload$(EXEEXT)
 	$(AM_V_CXXLD)$(CXXLINK) $(src_tools_linux_symupload_minidump_upload_OBJECTS) $(src_tools_linux_symupload_minidump_upload_LDADD) $(LIBS)
+src/common/linux/symbol_collector_client.$(OBJEXT):  \
+	src/common/linux/$(am__dirstamp) \
+	src/common/linux/$(DEPDIR)/$(am__dirstamp)
 src/common/linux/symbol_upload.$(OBJEXT):  \
 	src/common/linux/$(am__dirstamp) \
 	src/common/linux/$(DEPDIR)/$(am__dirstamp)
@@ -4996,6 +5009,7 @@ distclean-compile:
 @AMDEP_TRUE@@am__include@ @am__quote@src/common/linux/$(DEPDIR)/src_tools_linux_dump_syms_dump_syms-linux_libc_support.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@src/common/linux/$(DEPDIR)/src_tools_linux_dump_syms_dump_syms-memory_mapped_file.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@src/common/linux/$(DEPDIR)/src_tools_linux_dump_syms_dump_syms-safe_readlink.Po@am__quote@
+@AMDEP_TRUE@@am__include@ @am__quote@src/common/linux/$(DEPDIR)/symbol_collector_client.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@src/common/linux/$(DEPDIR)/symbol_upload.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@src/common/linux/tests/$(DEPDIR)/src_client_linux_linux_client_unittest_shlib-crash_generator.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@src/common/linux/tests/$(DEPDIR)/src_common_dumper_unittest-crash_generator.Po@am__quote@

docs/sym_upload_v2_protocol.md

@@ -0,0 +1,214 @@
+# Introduction
+
+The `sym_upload` tool is able to operate in `sym-upload-v2` protocol mode, in
+addition to the legacy protocol (which will be referred to as `sym-upload-v1`
+for the rest of this document). For now `sym-upload-v2` is HTTP/REST-based but
+it could be extended to operate over gRPC instead, in the future.
+
+# Table of Contents
+* [Why](#why)
+* [How](#how)
+   * [Uploading](#uploading)
+     * [Uploading with `sym_upload`](#uploading-with-sym_upload)
+     * [Uploading with curl](#uploading-with-curl)
+     * [Serving the `sym-upload-v2` protocol](#serving-the-sym-upload-v2-protocol)
+       * [Authenticate using `key`](#authenticate-using-key)
+       * [Symbol `checkStatus`](#symbol-checkstatus)
+       * [Upload `create`](#upload-create)
+       * [Uploading the symbol file](#uploading-the-symbol-file)
+       * [Upload complete](#upload-complete)
+
+
+# Why
+
+Using `sym_upload` in `sym-upload-v2` protocol mode has the following features
+beyond `sym-upload-v1`:
+  * Authentication via `key` (arbitrary secret).
+  * Symbol identifier (product of `debug_file` and `debug_id`, as recorded in
+output from `dump_syms`) can be checked against existing symbol information on
+server. If it's present, then the upload is skipped entirely.
+
+# How
+
+## Uploading
+
+### Uploading with `sym_upload`
+
+Uploading in `sym-upload-v2` protocol mode is easy. Invoke `sym_upload` like
+```
+$ ./sym_upload -p sym-upload-v2 [-k <API-key>] <symbol-file> <API-URL>
+```
+
+Where `symbol-file` is a symbol file created by `dump_syms`, `API-URL` is the
+URL of your `sym-upload-v2` API service (see next section for details), and
+`API-key` is a secret known to your uploader and server.
+
+For more options see `sym_upload --help`.
+
+### Uploading with curl
+
+As an example, if:
+  * Your API's URL was "https://sym-upload-api".
+  * Your service has assigned you `key` "myfancysecret123".
+  * You wanted to upload the symbol file at "path/to/file_name", with
+`debug_file` being "file_name" and `debug_id` being
+"123123123123123123123123123". Normally you would read these values from
+"path/to/file_name", which in turn was generated by `dump_syms`.
+
+Then you might run:
+```
+$ curl https://sym-upload-api/symbols/file_name/123123123123123123123123123:checkStatus?key=myfancysecret123
+```
+
+And, upon seeing that this `debug_file`/`debug_id` combo is missing from symbol
+storage then you could run:
+```
+$ curl --request POST https://sym-upload-api/uploads:create?key=myfancysecret123
+```
+
+Which returns `upload_url` "https://upload-server/42?creds=shhhhh" and
+`upload_key` "42". Next you upload the file directly like:
+```
+$ curl -T path/to/file_name "https://upload-server/42?creds=shhhhh"
+```
+
+Once the HTTP PUT is complete, run:
+```
+$ curl --header "Content-Type: application/json" \
+    --request POST \
+    --data '{symbol_id:{"debugFile":"file_name",'\
+        '"debugId":"123123123123123123123123123"}}' \
+    https://sym-upload-api/uploads/42:complete?key=myfancysecret123
+```
+
+### Serving the `sym-upload-v2` Protocol
+
+The protocol is currently defined only in HTTP/REST. There are three necessary
+REST operations to implement in your service:
+* `/symbols/<debug_file>/<debug_id>:checkStatus?key=<key>`
+* `/uploads:create?key=<key>`
+* `/uploads/<upload_key>:complete?key=<key>`
+
+#### Authenticate Using `key`
+
+The query string arg `key` contains some secret that both the uploader and
+server understand. It is up to the service implementer to decide on what
+constitutes a valid `key`, how the uploader acquires one, and how to handle
+requests made with invalid ones.
+
+#### Symbol `checkStatus`
+
+```
+/symbols/<debug_file>/<debug_id>:checkStatus?key=<key>
+```
+
+This operation expects an empty (or no) JSON payload in the request.
+
+This operation should return the status of the symbol file uniquely identified
+by the given `debug_file` and `debug_id`. JSON schema:
+```
+{
+    "type": object",
+    "properties": {
+        "status": {
+            "type": "string",
+            "enum": ["STATUS_UNSPECIFIED", "MISING", "FOUND"],
+            "required": true
+        }
+    }
+}
+```
+
+Where `MISSING` denotes that the symbol file does not exist on the server and
+`FOUND` denotes that the symbol file exists on the server.
+
+#### Upload `create`
+
+```
+/uploads:create?key=<key>
+```
+
+This operation expects an empty (or no) JSON payload in the request.
+
+This operation should return a URL that uploader can HTTP PUT their symbol file
+to, along with an "upload key" that can be used to notify the service once the
+file upload is completed. JSON schema:
+```
+{
+    "type": "object",
+    "properties": {
+        "upload_url": {
+            "type: "string",
+            "required": true
+        },
+        "upload_key": {
+            "type": "string",
+            "required": true
+        }
+    }
+}
+```
+
+Since this REST API operation can be authenticated via the `key` query string
+arg, the service can return a URL that encodes permission delegation to the
+upload endpoint resource and thereby constrain the ability to upload to those
+with valid `key`s.
+
+#### Uploading the Symbol File
+
+Note that the actual symbol upload step is _not_ part of the REST API. The
+upload URL obtained in the above operation is meant to be used as the endpoint
+for a normal HTTP PUT request for the contents of the symbol file. Once that
+HTTP PUT request is completed use the upload `complete` operation.
+
+#### Upload `complete`
+
+```
+/uploads/<upload_key>:complete?key=<key>
+```
+
+This operation expects a JSON payload in the HTTP request body with the
+following schema:
+```
+{
+    "type": "object",
+    "properties": {
+        "symbol_id": {
+            "type": "object",
+            "properties": {
+                "debug_file": {
+                    "type": "string",
+                    "required": true
+                },
+                "debug_id": {
+                    "type": "string",
+                    "required": true
+                }
+            }
+        }
+    }
+}
+```
+
+This operation should cause the symbol storage back-end (however implemented)
+to consume the symbol file identified by `upload_key`. It is up to the service
+implementation to decide how uploads are assigned `upload_key`s and how to
+retrieve a completed upload by its `upload_key`. If the symbol file cannot be
+found, is malformed, or the operation cannot be completed for any other reason
+then an HTTP error will be returned. JSON schema of non-error responses:
+```
+{
+    "type": "object",
+    "properties": {
+        "result": {
+            "type": string,
+            "enum": ["RESULT_UNSPECIFIED", "OK", "DUPLICATE_DATA"],
+            "required": true
+        }
+    }
+}
+```
+
+Where `OK` denotes that the symbol storage was updated with the new symbol file
+and `DUPLICATE_DATA` denotes that the symbol file data was identical to data
+already in symbol storage and therefore nothing changed.

src/common/common.gyp

@@ -118,6 +118,8 @@
         'linux/memory_mapped_file.h',
         'linux/safe_readlink.cc',
         'linux/safe_readlink.h',
+        'linux/symbol_collector_client.cc',
+        'linux/symbol_collector_client.h',
         'linux/synth_elf.cc',
         'linux/synth_elf.h',
         'long_string_dictionary.cc',

src/common/linux/libcurl_wrapper.cc

@@ -38,32 +38,24 @@
 namespace google_breakpad {
 LibcurlWrapper::LibcurlWrapper()
     : init_ok_(false),
-      formpost_(NULL),
+      curl_lib_(nullptr),
-      lastptr_(NULL),
+      last_curl_error_(""),
-      headerlist_(NULL) {
+      curl_(nullptr),
-  curl_lib_ = dlopen("libcurl.so", RTLD_NOW);
+      formpost_(nullptr),
-  if (!curl_lib_) {
+      lastptr_(nullptr),
-    curl_lib_ = dlopen("libcurl.so.4", RTLD_NOW);
+      headerlist_(nullptr) {}
-  }
-  if (!curl_lib_) {
-    curl_lib_ = dlopen("libcurl.so.3", RTLD_NOW);
-  }
-  if (!curl_lib_) {
-    std::cout << "Could not find libcurl via dlopen";
-    return;
-  }
-  std::cout << "LibcurlWrapper init succeeded";
-  init_ok_ = true;
-  return;
-}
 
-LibcurlWrapper::~LibcurlWrapper() {}
+LibcurlWrapper::~LibcurlWrapper() {
+  if (init_ok_) {
+    (*easy_cleanup_)(curl_);
+    dlclose(curl_lib_);
+  }
+}
 
 bool LibcurlWrapper::SetProxy(const string& proxy_host,
                               const string& proxy_userpwd) {
-  if (!init_ok_) {
+  if (!CheckInit()) return false;
-    return false;
+
-  }
   // Set proxy information if necessary.
   if (!proxy_host.empty()) {
     (*easy_setopt_)(curl_, CURLOPT_PROXY, proxy_host.c_str());
@@ -83,9 +75,8 @@ bool LibcurlWrapper::SetProxy(const string& proxy_host,
 
 bool LibcurlWrapper::AddFile(const string& upload_file_path,
                              const string& basename) {
-  if (!init_ok_) {
+  if (!CheckInit()) return false;
-    return false;
+
-  }
   std::cout << "Adding " << upload_file_path << " to form upload.";
   // Add form file.
   (*formadd_)(&formpost_, &lastptr_,
@@ -110,10 +101,11 @@ static size_t WriteCallback(void *ptr, size_t size,
 
 bool LibcurlWrapper::SendRequest(const string& url,
                                  const std::map<string, string>& parameters,
-                                 int* http_status_code,
+                                 long* http_status_code,
                                  string* http_header_data,
                                  string* http_response_data) {
-  (*easy_setopt_)(curl_, CURLOPT_URL, url.c_str());
+  if (!CheckInit()) return false;
+
   std::map<string, string>::const_iterator iter = parameters.begin();
   for (; iter != parameters.end(); ++iter)
     (*formadd_)(&formpost_, &lastptr_,
@@ -122,55 +114,79 @@ bool LibcurlWrapper::SendRequest(const string& url,
                 CURLFORM_END);
 
   (*easy_setopt_)(curl_, CURLOPT_HTTPPOST, formpost_);
-  if (http_response_data != NULL) {
+
-    http_response_data->clear();
+  return SendRequestInner(url, http_status_code, http_header_data,
-    (*easy_setopt_)(curl_, CURLOPT_WRITEFUNCTION, WriteCallback);
+                          http_response_data);
-    (*easy_setopt_)(curl_, CURLOPT_WRITEDATA,
+}
-                     reinterpret_cast<void *>(http_response_data));
+
-  }
+bool LibcurlWrapper::SendGetRequest(const string& url,
-  if (http_header_data != NULL) {
+                                    long* http_status_code,
-    http_header_data->clear();
+                                    string* http_header_data,
-    (*easy_setopt_)(curl_, CURLOPT_HEADERFUNCTION, WriteCallback);
+                                    string* http_response_data) {
-    (*easy_setopt_)(curl_, CURLOPT_HEADERDATA,
+  if (!CheckInit()) return false;
-                     reinterpret_cast<void *>(http_header_data));
+
+  (*easy_setopt_)(curl_, CURLOPT_HTTPGET, 1L);
+
+  return SendRequestInner(url, http_status_code, http_header_data,
+                          http_response_data);
+}
+
+bool LibcurlWrapper::SendPutRequest(const string& url,
+                                    const string& path,
+                                    long* http_status_code,
+                                    string* http_header_data,
+                                    string* http_response_data) {
+  if (!CheckInit()) return false;
+
+  FILE* file = fopen(path.c_str(), "rb");
+  (*easy_setopt_)(curl_, CURLOPT_UPLOAD, 1L);
+  (*easy_setopt_)(curl_, CURLOPT_PUT, 1L);
+  (*easy_setopt_)(curl_, CURLOPT_READDATA, file);
+
+  bool success = SendRequestInner(url, http_status_code, http_header_data,
+                                  http_response_data);
+
+  fclose(file);
+  return success;
+}
+
+bool LibcurlWrapper::SendSimplePostRequest(const string& url,
+                                           const string& body,
+                                           const string& content_type,
+                                           long* http_status_code,
+                                           string* http_header_data,
+                                           string* http_response_data) {
+  if (!CheckInit()) return false;
+
+  (*easy_setopt_)(curl_, CURLOPT_POSTFIELDSIZE, body.size());
+  (*easy_setopt_)(curl_, CURLOPT_COPYPOSTFIELDS, body.c_str());
+
+  if (!content_type.empty()) {
+    string content_type_header = "Content-Type: " + content_type;
+    headerlist_ = (*slist_append_)(
+        headerlist_,
+        content_type_header.c_str());
   }
 
-  CURLcode err_code = CURLE_OK;
+  return SendRequestInner(url, http_status_code, http_header_data,
-  err_code = (*easy_perform_)(curl_);
+                          http_response_data);
-  easy_strerror_ = reinterpret_cast<const char* (*)(CURLcode)>
-                       (dlsym(curl_lib_, "curl_easy_strerror"));
-
-  if (http_status_code != NULL) {
-    (*easy_getinfo_)(curl_, CURLINFO_RESPONSE_CODE, http_status_code);
-  }
-
-#ifndef NDEBUG
-  if (err_code != CURLE_OK)
-    fprintf(stderr, "Failed to send http request to %s, error: %s\n",
-            url.c_str(),
-            (*easy_strerror_)(err_code));
-#endif
-  if (headerlist_ != NULL) {
-    (*slist_free_all_)(headerlist_);
-  }
-
-  (*easy_cleanup_)(curl_);
-  if (formpost_ != NULL) {
-    (*formfree_)(formpost_);
-  }
-
-  return err_code == CURLE_OK;
 }
 
 bool LibcurlWrapper::Init() {
-  if (!init_ok_) {
+  curl_lib_ = dlopen("libcurl.so", RTLD_NOW);
-    std::cout << "Init_OK was not true in LibcurlWrapper::Init(), check earlier log messages";
+  if (!curl_lib_) {
+    curl_lib_ = dlopen("libcurl.so.4", RTLD_NOW);
+  }
+  if (!curl_lib_) {
+    curl_lib_ = dlopen("libcurl.so.3", RTLD_NOW);
+  }
+  if (!curl_lib_) {
+    std::cout << "Could not find libcurl via dlopen";
     return false;
   }
 
   if (!SetFunctionPointers()) {
     std::cout << "Could not find function pointers";
-    init_ok_ = false;
     return false;
   }
 
@@ -184,11 +200,7 @@ bool LibcurlWrapper::Init() {
     return false;
   }
 
-  // Disable 100-continue header.
+  init_ok_ = true;
-  char buf[] = "Expect:";
-
-  headerlist_ = (*slist_append_)(headerlist_, buf);
-  (*easy_setopt_)(curl_, CURLOPT_HTTPHEADER, headerlist_);
   return true;
 }
 
@@ -228,6 +240,10 @@ bool LibcurlWrapper::SetFunctionPointers() {
                                  "curl_easy_getinfo",
                                  CURLcode(*)(CURL *, CURLINFO info, ...));
 
+  SET_AND_CHECK_FUNCTION_POINTER(easy_reset_,
+                                 "curl_easy_reset",
+                                 void(*)(CURL*));
+
   SET_AND_CHECK_FUNCTION_POINTER(slist_free_all_,
                                  "curl_slist_free_all",
                                  void(*)(curl_slist*));
@@ -238,4 +254,73 @@ bool LibcurlWrapper::SetFunctionPointers() {
   return true;
 }
 
+bool LibcurlWrapper::SendRequestInner(const string& url,
+                                      long* http_status_code,
+                                      string* http_header_data,
+                                      string* http_response_data) {
+  string url_copy(url);
+  (*easy_setopt_)(curl_, CURLOPT_URL, url_copy.c_str());
+
+  // Disable 100-continue header.
+  char buf[] = "Expect:";
+  headerlist_ = (*slist_append_)(headerlist_, buf);
+  (*easy_setopt_)(curl_, CURLOPT_HTTPHEADER, headerlist_);
+
+  if (http_response_data != nullptr) {
+    http_response_data->clear();
+    (*easy_setopt_)(curl_, CURLOPT_WRITEFUNCTION, WriteCallback);
+    (*easy_setopt_)(curl_, CURLOPT_WRITEDATA,
+                    reinterpret_cast<void*>(http_response_data));
+  }
+  if (http_header_data != nullptr) {
+    http_header_data->clear();
+    (*easy_setopt_)(curl_, CURLOPT_HEADERFUNCTION, WriteCallback);
+    (*easy_setopt_)(curl_, CURLOPT_HEADERDATA,
+                    reinterpret_cast<void*>(http_header_data));
+  }
+  CURLcode err_code = CURLE_OK;
+  err_code = (*easy_perform_)(curl_);
+  easy_strerror_ = reinterpret_cast<const char* (*)(CURLcode)>
+      (dlsym(curl_lib_, "curl_easy_strerror"));
+
+  if (http_status_code != nullptr) {
+    (*easy_getinfo_)(curl_, CURLINFO_RESPONSE_CODE, http_status_code);
+  }
+
+#ifndef NDEBUG
+  if (err_code != CURLE_OK)
+    fprintf(stderr, "Failed to send http request to %s, error: %s\n",
+            url.c_str(),
+            (*easy_strerror_)(err_code));
+#endif
+
+  Reset();
+
+  return err_code == CURLE_OK;
 }
+
+void LibcurlWrapper::Reset() {
+  if (headerlist_ != nullptr) {
+    (*slist_free_all_)(headerlist_);
+    headerlist_ = nullptr;
+  }
+
+  if (formpost_ != nullptr) {
+    (*formfree_)(formpost_);
+    formpost_ = nullptr;
+  }
+
+  (*easy_reset_)(curl_);
+}
+
+bool LibcurlWrapper::CheckInit() {
+  if (!init_ok_) {
+    std::cout << "LibcurlWrapper: You must call Init(), and have it return "
+                 "'true' before invoking any other methods.\n";
+    return false;
+  }
+
+  return true;
+}
+
+}  // namespace google_breakpad

src/common/linux/libcurl_wrapper.h

@@ -51,14 +51,39 @@ class LibcurlWrapper {
                        const string& basename);
   virtual bool SendRequest(const string& url,
                            const std::map<string, string>& parameters,
-                           int* http_status_code,
+                           long* http_status_code,
                            string* http_header_data,
                            string* http_response_data);
+  bool SendGetRequest(const string& url,
+                      long* http_status_code,
+                      string* http_header_data,
+                      string* http_response_data);
+  bool SendPutRequest(const string& url,
+                      const string& path,
+                      long* http_status_code,
+                      string* http_header_data,
+                      string* http_response_data);
+  bool SendSimplePostRequest(const string& url,
+                             const string& body,
+                             const string& content_type,
+                             long* http_status_code,
+                             string* http_header_data,
+                             string* http_response_data);
+
  private:
   // This function initializes class state corresponding to function
   // pointers into the CURL library.
   bool SetFunctionPointers();
 
+  bool SendRequestInner(const string& url,
+                        long* http_status_code,
+                        string* http_header_data,
+                        string* http_response_data);
+
+  void Reset();
+
+  bool CheckInit();
+
   bool init_ok_;                 // Whether init succeeded
   void* curl_lib_;               // Pointer to result of dlopen() on
                                  // curl library
@@ -85,6 +110,7 @@ class LibcurlWrapper {
   const char* (*easy_strerror_)(CURLcode);
   void (*easy_cleanup_)(CURL *);
   CURLcode (*easy_getinfo_)(CURL *, CURLINFO info, ...);
+  void (*easy_reset_)(CURL*);
   void (*formfree_)(struct curl_httppost *);
 
 };

src/common/linux/symbol_collector_client.cc

@@ -0,0 +1,193 @@
+// Copyright (c) 2019 Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include "common/linux/symbol_collector_client.h"
+
+#include <stdio.h>
+
+#include <iostream>
+#include <regex>
+
+#include "common/linux/libcurl_wrapper.h"
+
+namespace google_breakpad {
+namespace sym_upload {
+
+// static
+bool SymbolCollectorClient::CreateUploadUrl(
+    LibcurlWrapper* libcurl_wrapper,
+    const string& api_url,
+    const string& api_key,
+    UploadUrlResponse* uploadUrlResponse) {
+  string header, response;
+  long response_code;
+
+  string url = api_url + "/v1/uploads:create";
+  if (!api_key.empty()) {
+    url += "?key=" + api_key;
+  }
+
+  if (!libcurl_wrapper->SendSimplePostRequest(url,
+                                              /*body=*/"",
+                                              /*content_type=*/"",
+                                              &response_code,
+                                              &header,
+                                              &response)) {
+    printf("Failed to create upload url.\n");
+    printf("Response code: %ld\n", response_code);
+    printf("Response:\n");
+    printf("%s\n", response.c_str());
+    return false;
+  }
+
+  // Note camel-case rather than underscores.
+  std::regex upload_url_regex("\"uploadUrl\": \"([^\"]+)\"");
+  std::regex upload_key_regex("\"uploadKey\": \"([^\"]+)\"");
+
+  std::smatch upload_url_match;
+  if (!std::regex_search(response, upload_url_match, upload_url_regex) ||
+      upload_url_match.size() != 2) {
+    printf("Failed to parse create url response.");
+    printf("Response:\n");
+    printf("%s\n", response.c_str());
+    return false;
+  }
+  string upload_url = upload_url_match[1].str();
+
+  std::smatch upload_key_match;
+  if (!std::regex_search(response, upload_key_match, upload_key_regex) ||
+      upload_key_match.size() != 2) {
+    printf("Failed to parse create url response.");
+    printf("Response:\n");
+    printf("%s\n", response.c_str());
+    return false;
+  }
+  string upload_key = upload_key_match[1].str();
+
+  uploadUrlResponse->upload_url = upload_url;
+  uploadUrlResponse->upload_key = upload_key;
+  return true;
+}
+
+// static
+CompleteUploadResult SymbolCollectorClient::CompleteUpload(
+    LibcurlWrapper* libcurl_wrapper,
+    const string& api_url,
+    const string& api_key,
+    const string& upload_key,
+    const string& debug_file,
+    const string& debug_id) {
+  string header, response;
+  long response_code;
+
+  string url = api_url + "/v1/uploads/" + upload_key + ":complete";
+  if (!api_key.empty()) {
+    url += "?key=" + api_key;
+  }
+  string body =
+      "{ symbol_id: {"
+      "debug_file: \"" + debug_file + "\", "
+      "debug_id: \"" + debug_id + "\" } }";
+
+  if (!libcurl_wrapper->SendSimplePostRequest(url,
+                                              body,
+                                              "application/son",
+                                              &response_code,
+                                              &header,
+                                              &response)) {
+    printf("Failed to complete upload.\n");
+    printf("Response code: %ld\n", response_code);
+    printf("Response:\n");
+    printf("%s\n", response.c_str());
+    return CompleteUploadResult::Error;
+  }
+
+  std::regex result_regex("\"result\": \"([^\"]+)\"");
+  std::smatch result_match;
+  if (!std::regex_search(response, result_match, result_regex) ||
+      result_match.size() != 2) {
+    printf("Failed to parse complete upload response.");
+    printf("Response:\n");
+    printf("%s\n", response.c_str());
+    return CompleteUploadResult::Error;
+  }
+  string result = result_match[1].str();
+
+  if (result.compare("DUPLICATE_DATA") == 0) {
+    return CompleteUploadResult::DuplicateData;
+  }
+
+  return CompleteUploadResult::Ok;
+}
+
+// static
+SymbolStatus SymbolCollectorClient::CheckSymbolStatus(
+    LibcurlWrapper* libcurl_wrapper,
+    const string& api_url,
+    const string& api_key,
+    const string& debug_file,
+    const string& debug_id) {
+  string header, response;
+  long response_code;
+  string url = api_url +
+               "/v1/symbols/" + debug_file + "/" + debug_id + ":checkStatus";
+  if (!api_key.empty()) {
+    url += "?key=" + api_key;
+  }
+
+  if (!libcurl_wrapper->SendGetRequest(
+      url,
+      &response_code,
+      &header,
+      &response)) {
+    printf("Failed to check symbol status, error message.\n");
+    printf("Response code: %ld\n", response_code);
+    printf("Response:\n");
+    printf("%s\n", response.c_str());
+    return SymbolStatus::Unknown;
+  }
+
+  std::regex status_regex("\"status\": \"([^\"]+)\"");
+  std::smatch status_match;
+  if (!std::regex_search(response, status_match, status_regex) ||
+      status_match.size() != 2) {
+    printf("Failed to parse check symbol status response.");
+    printf("Response:\n");
+    printf("%s\n", response.c_str());
+    return SymbolStatus::Unknown;
+  }
+  string status = status_match[1].str();
+
+  return (status.compare("FOUND") == 0) ?
+      SymbolStatus::Found :
+      SymbolStatus::Missing;
+}
+
+}  // namespace sym_upload
+}  // namespace google_breakpad

src/common/linux/symbol_collector_client.h

@@ -0,0 +1,87 @@
+// Copyright (c) 2019, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#ifndef COMMON_LINUX_SYMBOL_COLLECTOR_CLIENT_H_
+#define COMMON_LINUX_SYMBOL_COLLECTOR_CLIENT_H_
+
+#include <string>
+
+#include "common/linux/libcurl_wrapper.h"
+#include "common/using_std_string.h"
+
+namespace google_breakpad {
+namespace sym_upload {
+
+struct UploadUrlResponse {
+  string upload_url;
+  string upload_key;
+};
+
+enum SymbolStatus {
+  Found,
+  Missing,
+  Unknown
+};
+
+enum CompleteUploadResult {
+  Ok,
+  DuplicateData,
+  Error
+};
+
+// Helper class to communicate with a sym-upload-v2 service over HTTP/REST,
+// via libcurl.
+class SymbolCollectorClient {
+ public:
+  static bool CreateUploadUrl(
+      LibcurlWrapper* libcurl_wrapper,
+      const string& api_url,
+      const string& api_key,
+      UploadUrlResponse* uploadUrlResponse);
+
+  static CompleteUploadResult CompleteUpload(
+      LibcurlWrapper* libcurl_wrapper,
+      const string& api_url,
+      const string& api_key,
+      const string& upload_key,
+      const string& debug_file,
+      const string& debug_id);
+
+  static SymbolStatus CheckSymbolStatus(
+      LibcurlWrapper* libcurl_wrapper,
+      const string& api_url,
+      const string& api_key,
+      const string& debug_file,
+      const string& debug_id);
+};
+
+}  // namespace sym_upload
+}  // namespace google_breakpad
+
+#endif  // COMMON_LINUX_SYMBOL_COLLECTOR_CLIENT_H_

src/common/linux/symbol_upload.cc

@@ -30,15 +30,19 @@
 // symbol_upload.cc: implemented google_breakpad::sym_upload::Start, a helper
 // function for linux symbol upload tool.
 
-#include "common/linux/http_upload.h"
 #include "common/linux/symbol_upload.h"
 
 #include <assert.h>
 #include <stdio.h>
 
 #include <functional>
+#include <iostream>
 #include <vector>
 
+#include "common/linux/http_upload.h"
+#include "common/linux/libcurl_wrapper.h"
+#include "common/linux/symbol_collector_client.h"
+
 namespace google_breakpad {
 namespace sym_upload {
 
@@ -95,21 +99,19 @@ string CompactIdentifier(const string &uuid) {
   return result;
 }
 
-//=============================================================================
+// |options| describes the current sym_upload options.
-void Start(Options *options) {
+// |module_parts| contains the strings parsed from the MODULE entry of the
+// Breakpad symbol file being uploaded.
+// |compacted_id| is the debug_id from the MODULE entry of the Breakpad symbol
+// file being uploaded, with all hyphens removed.
+bool SymUploadV1Start(
+    const Options& options,
+    std::vector<string> module_parts,
+    const string& compacted_id) {
   std::map<string, string> parameters;
-  options->success = false;
-  std::vector<string> module_parts;
-  if (!ModuleDataForSymbolFile(options->symbolsPath, &module_parts)) {
-    fprintf(stderr, "Failed to parse symbol file!\n");
-    return;
-  }
-
-  string compacted_id = CompactIdentifier(module_parts[3]);
-
   // Add parameters
-  if (!options->version.empty())
+  if (!options.version.empty())
-    parameters["version"] = options->version;
+    parameters["version"] = options.version;
 
   // MODULE <os> <cpu> <uuid> <module-name>
   // 0      1    2     3      4
@@ -120,16 +122,16 @@ void Start(Options *options) {
   parameters["debug_identifier"] = compacted_id;
 
   std::map<string, string> files;
-  files["symbol_file"] = options->symbolsPath;
+  files["symbol_file"] = options.symbolsPath;
 
   string response, error;
   long response_code;
-  bool success = HTTPUpload::SendRequest(options->uploadURLStr,
+  bool success = HTTPUpload::SendRequest(options.uploadURLStr,
                                          parameters,
                                          files,
-                                         options->proxy,
+                                         options.proxy,
-                                         options->proxy_user_pwd,
+                                         options.proxy_user_pwd,
-                                         "",
+                                         /*ca_certificate_file=*/"",
                                          &response,
                                          &response_code,
                                          &error);
@@ -148,7 +150,116 @@ void Start(Options *options) {
   } else {
     printf("Successfully sent the symbol file.\n");
   }
-  options->success = success;
+
+  return success;
+}
+
+// |options| describes the current sym_upload options.
+// |module_parts| contains the strings parsed from the MODULE entry of the
+// Breakpad symbol file being uploaded.
+// |compacted_id| is the debug_id from the MODULE entry of the Breakpad symbol
+// file being uploaded, with all hyphens removed.
+bool SymUploadV2Start(
+    const Options& options,
+    std::vector<string> module_parts,
+    const string& compacted_id) {
+  string debug_file = module_parts[4];
+  string debug_id = compacted_id;
+
+  google_breakpad::LibcurlWrapper libcurl_wrapper;
+  if (!libcurl_wrapper.Init()) {
+    printf("Failed to init google_breakpad::LibcurlWrapper.\n");
+    return false;
+  }
+
+  if (!options.force) {
+    SymbolStatus symbolStatus = SymbolCollectorClient::CheckSymbolStatus(
+        &libcurl_wrapper,
+        options.uploadURLStr,
+        options.api_key,
+        debug_file,
+        debug_id);
+    if (symbolStatus == SymbolStatus::Found) {
+      printf("Symbol file already exists, upload aborted."
+          " Use \"-f\" to overwrite.\n");
+      return true;
+    } else if (symbolStatus == SymbolStatus::Unknown) {
+      printf("Failed to check for existing symbol.\n");
+      return false;
+    }
+  }
+
+  UploadUrlResponse uploadUrlResponse;
+  if (!SymbolCollectorClient::CreateUploadUrl(
+      &libcurl_wrapper,
+      options.uploadURLStr,
+      options.api_key,
+      &uploadUrlResponse)) {
+    printf("Failed to create upload URL.\n");
+    return false;
+  }
+
+  string signed_url = uploadUrlResponse.upload_url;
+  string upload_key = uploadUrlResponse.upload_key;
+  string header;
+  string response;
+  long response_code;
+
+  if (!libcurl_wrapper.SendPutRequest(signed_url,
+                                      options.symbolsPath,
+                                      &response_code,
+                                      &header,
+                                      &response)) {
+    printf("Failed to send symbol file.\n");
+    printf("Response code: %ld\n", response_code);
+    printf("Response:\n");
+    printf("%s\n", response.c_str());
+    return false;
+  } else if (response_code == 0) {
+    printf("Failed to send symbol file: No response code\n");
+    return false;
+  } else if (response_code != 200) {
+    printf("Failed to send symbol file: Response code %ld\n", response_code);
+    printf("Response:\n");
+    printf("%s\n", response.c_str());
+    return false;
+  }
+
+  CompleteUploadResult completeUploadResult =
+      SymbolCollectorClient::CompleteUpload(&libcurl_wrapper,
+                                            options.uploadURLStr,
+                                            options.api_key,
+                                            upload_key,
+                                            debug_file,
+                                            debug_id);
+  if (completeUploadResult == CompleteUploadResult::Error) {
+    printf("Failed to complete upload.\n");
+    return false;
+  } else if (completeUploadResult == CompleteUploadResult::DuplicateData) {
+    printf("Uploaded file checksum matched existing file checksum,"
+      " no change necessary.\n");
+  } else {
+    printf("Successfully sent the symbol file.\n");
+  }
+
+  return true;
+}
+
+//=============================================================================
+void Start(Options* options) {
+  std::vector<string> module_parts;
+  if (!ModuleDataForSymbolFile(options->symbolsPath, &module_parts)) {
+    fprintf(stderr, "Failed to parse symbol file!\n");
+    return;
+  }
+
+  const string compacted_id = CompactIdentifier(module_parts[3]);
+
+  if (options->upload_protocol == UploadProtocol::SYM_UPLOAD_V2) {
+    options->success = SymUploadV2Start(*options, module_parts, compacted_id);
+  } else {
+    options->success = SymUploadV1Start(*options, module_parts, compacted_id);
+  }
 }
 
 }  // namespace sym_upload

src/common/linux/symbol_upload.h

@@ -41,14 +41,24 @@
 namespace google_breakpad {
 namespace sym_upload {
 
-typedef struct {
+enum class UploadProtocol {
+  SYM_UPLOAD_V1,
+  SYM_UPLOAD_V2,
+};
+
+struct Options {
+  Options() : upload_protocol(UploadProtocol::SYM_UPLOAD_V1), force(false) {}
+
   string symbolsPath;
   string uploadURLStr;
   string proxy;
   string proxy_user_pwd;
   string version;
   bool success;
-} Options;
+  UploadProtocol upload_protocol;
+  bool force;
+  string api_key;
+};
 
 // Starts upload to symbol server with options.
 void Start(Options* options);

src/tools/linux/symupload/sym_upload.cc

@@ -41,25 +41,43 @@
 
 #include <stdio.h>
 #include <stdlib.h>
+#include <string.h>
 #include <unistd.h>
 
 #include "common/linux/symbol_upload.h"
 
+using google_breakpad::sym_upload::UploadProtocol;
 using google_breakpad::sym_upload::Options;
 
 //=============================================================================
 static void
 Usage(int argc, const char *argv[]) {
   fprintf(stderr, "Submit symbol information.\n");
-  fprintf(stderr, "Usage: %s [options...] <symbols> <upload-URL>\n", argv[0]);
+  fprintf(stderr, "Usage: %s [options...] <symbol-file> <upload-URL>\n",
+      argv[0]);
   fprintf(stderr, "Options:\n");
-  fprintf(stderr, "<symbols> should be created by using the dump_syms tool.\n");
+  fprintf(stderr, "<symbol-file> should be created by using the dump_syms"
+      "tool.\n");
   fprintf(stderr, "<upload-URL> is the destination for the upload\n");
+  fprintf(stderr, "-p:\t <protocol> One of ['sym-upload-v1',"
+    " 'sym-upload-v2'], defaults to 'sym-upload-v1'.\n");
+  fprintf(stderr, "-k:\t <API-key> A secret used to authenticate with the"
+      " API [Only supported when using 'sym-upload-v2' protocol].\n");
+  fprintf(stderr, "-f:\t Force symbol upload if already exists [Only"
+      " supported when using 'sym-upload-v2' protocol].\n");
   fprintf(stderr, "-v:\t Version information (e.g., 1.2.3.4)\n");
   fprintf(stderr, "-x:\t <host[:port]> Use HTTP proxy on given port\n");
   fprintf(stderr, "-u:\t <user[:password]> Set proxy user and password\n");
   fprintf(stderr, "-h:\t Usage\n");
   fprintf(stderr, "-?:\t Usage\n");
+  fprintf(stderr, "\n");
+  fprintf(stderr, "Examples:\n");
+  fprintf(stderr, "  With 'sym-upload-v1':\n");
+  fprintf(stderr, "    %s path/to/symbol_file http://myuploadserver\n",
+      argv[0]);
+  fprintf(stderr, "  With 'sym-upload-v2':\n");
+  fprintf(stderr, "    %s -p sym-upload-v2 -k mysecret123! "
+      "path/to/symbol_file http://myuploadserver\n", argv[0]);
 }
 
 //=============================================================================
@@ -68,7 +86,7 @@ SetupOptions(int argc, const char *argv[], Options *options) {
   extern int optind;
   int ch;
 
-  while ((ch = getopt(argc, (char * const *)argv, "u:v:x:h?")) != -1) {
+  while ((ch = getopt(argc, (char * const *)argv, "u:v:x:p:k:hf?")) != -1) {
     switch (ch) {
       case 'h':
       case '?':
@@ -84,6 +102,23 @@ SetupOptions(int argc, const char *argv[], Options *options) {
       case 'x':
         options->proxy = optarg;
         break;
+      case 'p':
+        if (strcmp(optarg, "sym-upload-v2") == 0) {
+          options->upload_protocol = UploadProtocol::SYM_UPLOAD_V2;
+        } else if (strcmp(optarg, "sym-upload-v1") == 0) {
+          options->upload_protocol = UploadProtocol::SYM_UPLOAD_V1;
+        } else {
+          fprintf(stderr, "Invalid protocol '%c'\n", optarg);
+          Usage(argc, argv);
+          exit(1);
+        }
+        break;
+      case 'k':
+        options->api_key = optarg;
+        break;
+      case 'f':
+        options->force = true;
+        break;
 
       default:
         fprintf(stderr, "Invalid option '%c'\n", ch);