Google Cloud Storage C++ Client  1.24.0
A C++ Client Library for Google Cloud Storage
client.cc
Go to the documentation of this file.
1 // Copyright 2018 Google LLC
2 //
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
6 //
7 // http://www.apache.org/licenses/LICENSE-2.0
8 //
9 // Unless required by applicable law or agreed to in writing, software
10 // distributed under the License is distributed on an "AS IS" BASIS,
11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 // See the License for the specific language governing permissions and
13 // limitations under the License.
14 
16 #include "google/cloud/storage/internal/curl_client.h"
17 #include "google/cloud/storage/internal/curl_handle.h"
18 #include "google/cloud/storage/internal/openssl_util.h"
20 #include "google/cloud/internal/filesystem.h"
21 #include "google/cloud/log.h"
22 #include "absl/memory/memory.h"
23 #include <openssl/md5.h>
24 #include <fstream>
25 #include <thread>
26 
27 namespace google {
28 namespace cloud {
29 namespace storage {
30 inline namespace STORAGE_CLIENT_NS {
31 static_assert(std::is_copy_constructible<storage::Client>::value,
32  "storage::Client must be constructible");
33 static_assert(std::is_copy_assignable<storage::Client>::value,
34  "storage::Client must be assignable");
35 
36 std::shared_ptr<internal::RawClient> Client::CreateDefaultInternalClient(
37  ClientOptions options) {
38  return internal::CurlClient::Create(std::move(options));
39 }
40 
41 StatusOr<Client> Client::CreateDefaultClient() {
42  auto opts = ClientOptions::CreateDefaultClientOptions();
43  if (!opts) {
44  return StatusOr<Client>(opts.status());
45  }
46  return StatusOr<Client>(Client(*opts));
47 }
48 
49 ObjectReadStream Client::ReadObjectImpl(
50  internal::ReadObjectRangeRequest const& request) {
51  auto source = raw_client_->ReadObject(request);
52  if (!source) {
53  ObjectReadStream error_stream(
54  absl::make_unique<internal::ObjectReadStreambuf>(
55  request, std::move(source).status()));
56  error_stream.setstate(std::ios::badbit | std::ios::eofbit);
57  return error_stream;
58  }
59  auto stream =
60  ObjectReadStream(absl::make_unique<internal::ObjectReadStreambuf>(
61  request, *std::move(source),
62  request.GetOption<ReadFromOffset>().value_or(0)));
63  (void)stream.peek();
64 #if !GOOGLE_CLOUD_CPP_HAVE_EXCEPTIONS
65  // Without exceptions the streambuf cannot report errors, so we have to
66  // manually update the status bits.
67  if (!stream.status().ok()) {
68  stream.setstate(std::ios::badbit | std::ios::eofbit);
69  }
70 #endif // GOOGLE_CLOUD_CPP_HAVE_EXCEPTIONS
71  return stream;
72 }
73 
74 ObjectWriteStream Client::WriteObjectImpl(
75  internal::ResumableUploadRequest const& request) {
76  auto session = raw_client_->CreateResumableSession(request);
77  if (!session) {
78  auto error = absl::make_unique<internal::ResumableUploadSessionError>(
79  std::move(session).status());
80 
81  ObjectWriteStream error_stream(
82  absl::make_unique<internal::ObjectWriteStreambuf>(
83  std::move(error), 0,
84  absl::make_unique<internal::NullHashValidator>()));
85  error_stream.setstate(std::ios::badbit | std::ios::eofbit);
86  error_stream.Close();
87  return error_stream;
88  }
89  return ObjectWriteStream(absl::make_unique<internal::ObjectWriteStreambuf>(
90  *std::move(session), raw_client_->client_options().upload_buffer_size(),
91  internal::CreateHashValidator(request)));
92 }
93 
94 bool Client::UseSimpleUpload(std::string const& file_name,
95  std::size_t& size) const {
96  auto status = google::cloud::internal::status(file_name);
97  if (!is_regular(status)) {
98  return false;
99  }
100  auto const fs = google::cloud::internal::file_size(file_name);
101  if (fs <= raw_client()->client_options().maximum_simple_upload_size()) {
102  size = static_cast<std::size_t>(fs);
103  return true;
104  }
105  return false;
106 }
107 
108 StatusOr<ObjectMetadata> Client::UploadFileSimple(
109  std::string const& file_name, std::size_t file_size,
110  internal::InsertObjectMediaRequest request) {
111  auto upload_offset = request.GetOption<UploadFromOffset>().value_or(0);
112  if (file_size < upload_offset) {
113  std::ostringstream os;
114  os << __func__ << "(" << request << ", " << file_name
115  << "): UploadFromOffset (" << upload_offset
116  << ") is bigger than the size of file source (" << file_size << ")";
117  return Status(StatusCode::kInvalidArgument, std::move(os).str());
118  }
119  auto upload_size = (std::min)(
120  request.GetOption<UploadLimit>().value_or(file_size - upload_offset),
121  file_size - upload_offset);
122 
123  std::ifstream is(file_name, std::ios::binary);
124  if (!is.is_open()) {
125  std::ostringstream os;
126  os << __func__ << "(" << request << ", " << file_name
127  << "): cannot open upload file source";
128  return Status(StatusCode::kNotFound, std::move(os).str());
129  }
130 
131  std::string payload(static_cast<std::size_t>(upload_size), char{});
132  is.seekg(upload_offset, std::ios::beg);
133  is.read(&payload[0], payload.size());
134  if (static_cast<std::size_t>(is.gcount()) < payload.size()) {
135  std::ostringstream os;
136  os << __func__ << "(" << request << ", " << file_name << "): Actual read ("
137  << is.gcount() << ") is smaller than upload_size (" << payload.size()
138  << ")";
139  return Status(StatusCode::kInternal, std::move(os).str());
140  }
141  is.close();
142  request.set_contents(std::move(payload));
143 
144  return raw_client_->InsertObjectMedia(request);
145 }
146 
147 StatusOr<ObjectMetadata> Client::UploadFileResumable(
148  std::string const& file_name,
149  google::cloud::storage::internal::ResumableUploadRequest request) {
150  auto upload_offset = request.GetOption<UploadFromOffset>().value_or(0);
151  auto status = google::cloud::internal::status(file_name);
152  if (!is_regular(status)) {
153  GCP_LOG(WARNING) << "Trying to upload " << file_name
154  << R"""( which is not a regular file.
155 This is often a problem because:
156  - Some non-regular files are infinite sources of data, and the load will
157  never complete.
158  - Some non-regular files can only be read once, and UploadFile() may need to
159  read the file more than once to compute the checksum and hashes needed to
160  preserve data integrity.
161 
162 Consider using UploadLimit option or Client::WriteObject(). You may also need to disable data
163 integrity checks using the DisableMD5Hash() and DisableCrc32cChecksum() options.
164 )""";
165  } else {
166  std::error_code size_err;
167  auto file_size = google::cloud::internal::file_size(file_name, size_err);
168  if (size_err) {
169  return Status(StatusCode::kNotFound, size_err.message());
170  }
171  if (file_size < upload_offset) {
172  std::ostringstream os;
173  os << __func__ << "(" << request << ", " << file_name
174  << "): UploadFromOffset (" << upload_offset
175  << ") is bigger than the size of file source (" << file_size << ")";
176  return Status(StatusCode::kInvalidArgument, std::move(os).str());
177  }
178 
179  auto upload_size = (std::min)(
180  request.GetOption<UploadLimit>().value_or(file_size - upload_offset),
181  file_size - upload_offset);
182  request.set_option(UploadContentLength(upload_size));
183  }
184  std::ifstream source(file_name, std::ios::binary);
185  if (!source.is_open()) {
186  std::ostringstream os;
187  os << __func__ << "(" << request << ", " << file_name
188  << "): cannot open upload file source";
189  return Status(StatusCode::kNotFound, std::move(os).str());
190  }
191  // We set its offset before passing it to `UploadStreamResumable` so we don't
192  // need to compute `UploadFromOffset` again.
193  source.seekg(upload_offset, std::ios::beg);
194  return UploadStreamResumable(source, request);
195 }
196 
197 // NOLINTNEXTLINE(readability-make-member-function-const)
198 StatusOr<ObjectMetadata> Client::UploadStreamResumable(
199  std::istream& source, internal::ResumableUploadRequest const& request) {
200  StatusOr<std::unique_ptr<internal::ResumableUploadSession>> session_status =
201  raw_client()->CreateResumableSession(request);
202  if (!session_status) {
203  return std::move(session_status).status();
204  }
205 
206  auto session = std::move(*session_status);
207  // How many bytes of the local file are uploaded to the GCS server.
208  auto server_size = session->next_expected_byte();
209  auto upload_limit = request.GetOption<UploadLimit>().value_or(
210  (std::numeric_limits<std::uint64_t>::max)());
211  // If `server_size == upload_limit`, we will upload an empty string and
212  // finalize the upload.
213  if (server_size > upload_limit) {
214  return Status(StatusCode::kOutOfRange,
215  "UploadLimit (" + std::to_string(upload_limit) +
216  ") is not bigger than the uploaded size (" +
217  std::to_string(server_size) + ") on GCS server");
218  }
219  source.seekg(server_size, std::ios::cur);
220 
221  // GCS requires chunks to be a multiple of 256KiB.
222  auto chunk_size = internal::UploadChunkRequest::RoundUpToQuantum(
223  raw_client()->client_options().upload_buffer_size());
224 
225  StatusOr<internal::ResumableUploadResponse> upload_response(
226  internal::ResumableUploadResponse{});
227  // We iterate while `source` is good, the upload size does not reach the
228  // `UploadLimit` and the retry policy has not been exhausted.
229  bool reach_upload_limit = false;
230  internal::ConstBufferSequence buffers(1);
231  std::vector<char> buffer(chunk_size);
232  while (!source.eof() && upload_response &&
233  !upload_response->payload.has_value() && !reach_upload_limit) {
234  // Read a chunk of data from the source file.
235  if (upload_limit - server_size <= chunk_size) {
236  // We don't want the `source_size` to exceed `upload_limit`.
237  chunk_size = static_cast<std::size_t>(upload_limit - server_size);
238  reach_upload_limit = true;
239  }
240  source.read(buffer.data(), buffer.size());
241  auto gcount = static_cast<std::size_t>(source.gcount());
242  bool final_chunk = (gcount < buffer.size()) || reach_upload_limit;
243  auto source_size = session->next_expected_byte() + gcount;
244  auto expected = source_size;
245  buffers[0] = internal::ConstBuffer{buffer.data(), gcount};
246  if (final_chunk) {
247  upload_response = session->UploadFinalChunk(buffers, source_size);
248  } else {
249  upload_response = session->UploadChunk(buffers);
250  }
251  if (!upload_response) {
252  return std::move(upload_response).status();
253  }
254  if (session->next_expected_byte() != expected) {
255  // Defensive programming: unless there is a bug, this should be dead code.
256  return Status(
257  StatusCode::kInternal,
258  "Unexpected last committed byte expected=" +
259  std::to_string(expected) +
260  " got=" + std::to_string(session->next_expected_byte()) +
261  ". This is a bug, please report it at "
262  "https://github.com/googleapis/google-cloud-cpp/issues/new");
263  }
264 
265  // We only update `server_size` when uploading is successful.
266  server_size = expected;
267  }
268 
269  if (!upload_response) {
270  return std::move(upload_response).status();
271  }
272 
273  return *std::move(upload_response->payload);
274 }
275 
276 Status Client::DownloadFileImpl(internal::ReadObjectRangeRequest const& request,
277  std::string const& file_name) {
278  auto report_error = [&request, file_name](char const* func, char const* what,
279  Status const& status) {
280  std::ostringstream msg;
281  msg << func << "(" << request << ", " << file_name << "): " << what
282  << " - status.message=" << status.message();
283  return Status(status.code(), std::move(msg).str());
284  };
285 
286  auto stream = ReadObjectImpl(request);
287  if (!stream.status().ok()) {
288  return report_error(__func__, "cannot open download source object",
289  stream.status());
290  }
291 
292  // Open the destination file, and immediate raise an exception on failure.
293  std::ofstream os(file_name, std::ios::binary);
294  if (!os.is_open()) {
295  return report_error(
296  __func__, "cannot open download destination file",
297  Status(StatusCode::kInvalidArgument, "ofstream::open()"));
298  }
299 
300  std::string buffer;
301  buffer.resize(raw_client_->client_options().download_buffer_size(), '\0');
302  do {
303  stream.read(&buffer[0], buffer.size());
304  os.write(buffer.data(), stream.gcount());
305  } while (os.good() && stream.good());
306  os.close();
307  if (!os.good()) {
308  return report_error(__func__, "cannot close download destination file",
309  Status(StatusCode::kUnknown, "ofstream::close()"));
310  }
311  if (!stream.status().ok()) {
312  return report_error(__func__, "error reading download source object",
313  stream.status());
314  }
315  return Status();
316 }
317 
318 // NOLINTNEXTLINE(readability-make-member-function-const)
319 std::string Client::SigningEmail(SigningAccount const& signing_account) {
320  if (signing_account.has_value()) {
321  return signing_account.value();
322  }
323  return raw_client()->client_options().credentials()->AccountEmail();
324 }
325 
326 StatusOr<Client::SignBlobResponseRaw> Client::SignBlobImpl(
327  SigningAccount const& signing_account, std::string const& string_to_sign) {
328  auto credentials = raw_client()->client_options().credentials();
329 
330  std::string signing_account_email = SigningEmail(signing_account);
331  // First try to sign locally.
332  auto signed_blob = credentials->SignBlob(signing_account, string_to_sign);
333  if (signed_blob) {
334  return SignBlobResponseRaw{credentials->KeyId(), *std::move(signed_blob)};
335  }
336 
337  // If signing locally fails that may be because the credentials do not
338  // support signing, or because the signing account is different than the
339  // credentials account. In either case, try to sign using the API.
340  internal::SignBlobRequest sign_request(
341  signing_account_email, internal::Base64Encode(string_to_sign), {});
342  auto response = raw_client()->SignBlob(sign_request);
343  if (!response) {
344  return response.status();
345  }
346  return SignBlobResponseRaw{response->key_id,
347  internal::Base64Decode(response->signed_blob)};
348 }
349 
350 StatusOr<std::string> Client::SignUrlV2(
351  internal::V2SignUrlRequest const& request) {
352  SigningAccount const& signing_account = request.signing_account();
353  auto signed_blob = SignBlobImpl(signing_account, request.StringToSign());
354  if (!signed_blob) {
355  return signed_blob.status();
356  }
357 
358  internal::CurlHandle curl;
359  auto encoded = internal::Base64Encode(signed_blob->signed_blob);
360  std::string signature = curl.MakeEscapedString(encoded).get();
361 
362  std::ostringstream os;
363  os << "https://storage.googleapis.com/" << request.bucket_name();
364  if (!request.object_name().empty()) {
365  os << '/' << curl.MakeEscapedString(request.object_name()).get();
366  }
367  os << "?GoogleAccessId=" << SigningEmail(signing_account)
368  << "&Expires=" << request.expiration_time_as_seconds().count()
369  << "&Signature=" << signature;
370 
371  return std::move(os).str();
372 }
373 
374 StatusOr<std::string> Client::SignUrlV4(internal::V4SignUrlRequest request) {
375  auto valid = request.Validate();
376  if (!valid.ok()) {
377  return valid;
378  }
379  request.AddMissingRequiredHeaders();
380  SigningAccount const& signing_account = request.signing_account();
381  auto signing_email = SigningEmail(signing_account);
382 
383  auto string_to_sign = request.StringToSign(signing_email);
384  auto signed_blob = SignBlobImpl(signing_account, string_to_sign);
385  if (!signed_blob) {
386  return signed_blob.status();
387  }
388 
389  std::string signature = internal::HexEncode(signed_blob->signed_blob);
390  internal::CurlHandle curl;
391  std::ostringstream os;
392  os << request.HostnameWithBucket();
393  for (auto& part : request.ObjectNameParts()) {
394  os << '/' << curl.MakeEscapedString(part).get();
395  }
396  os << "?" << request.CanonicalQueryString(signing_email)
397  << "&X-Goog-Signature=" << signature;
398 
399  return std::move(os).str();
400 }
401 
402 StatusOr<PolicyDocumentResult> Client::SignPolicyDocument(
403  internal::PolicyDocumentRequest const& request) {
404  SigningAccount const& signing_account = request.signing_account();
405  auto signing_email = SigningEmail(signing_account);
406 
407  auto string_to_sign = request.StringToSign();
408  auto base64_policy = internal::Base64Encode(string_to_sign);
409  auto signed_blob = SignBlobImpl(signing_account, base64_policy);
410  if (!signed_blob) {
411  return signed_blob.status();
412  }
413 
414  return PolicyDocumentResult{
415  signing_email, request.policy_document().expiration, base64_policy,
416  internal::Base64Encode(signed_blob->signed_blob)};
417 }
418 
419 StatusOr<PolicyDocumentV4Result> Client::SignPolicyDocumentV4(
420  internal::PolicyDocumentV4Request request) {
421  SigningAccount const& signing_account = request.signing_account();
422  auto signing_email = SigningEmail(signing_account);
423  request.SetSigningEmail(signing_email);
424 
425  auto string_to_sign = request.StringToSign();
426  auto escaped = internal::PostPolicyV4Escape(string_to_sign);
427  if (!escaped) return escaped.status();
428  auto base64_policy = internal::Base64Encode(*escaped);
429  auto signed_blob = SignBlobImpl(signing_account, base64_policy);
430  if (!signed_blob) {
431  return signed_blob.status();
432  }
433  std::string signature = internal::HexEncode(signed_blob->signed_blob);
434  auto required_fiels = request.RequiredFormFields();
435  required_fiels["x-goog-signature"] = signature;
436  required_fiels["policy"] = base64_policy;
437  return PolicyDocumentV4Result{request.Url(),
438  request.Credentials(),
439  request.ExpirationDate(),
440  base64_policy,
441  signature,
442  "GOOG4-RSA-SHA256",
443  std::move(required_fiels)};
444 }
445 
446 std::string CreateRandomPrefixName(std::string const& prefix) {
447  auto constexpr kPrefixNameSize = 16;
448  auto rng = google::cloud::internal::MakeDefaultPRNG();
449  return prefix + google::cloud::internal::Sample(rng, kPrefixNameSize,
450  "abcdefghijklmnopqrstuvwxyz");
451 }
452 
453 namespace internal {
454 
455 ScopedDeleter::ScopedDeleter(
456  std::function<Status(std::string, std::int64_t)> delete_fun)
457  : enabled_(true), delete_fun_(std::move(delete_fun)) {}
458 
459 ScopedDeleter::~ScopedDeleter() {
460  if (enabled_) {
461  ExecuteDelete();
462  }
463 }
464 
465 void ScopedDeleter::Add(ObjectMetadata const& object) {
466  auto generation = object.generation();
467  Add(std::move(object).name(), generation);
468 }
469 
470 void ScopedDeleter::Add(std::string object_name, std::int64_t generation) {
471  object_list_.emplace_back(std::move(object_name), generation);
472 }
473 
474 Status ScopedDeleter::ExecuteDelete() {
475  std::vector<std::pair<std::string, std::int64_t>> object_list;
476  // make sure the dtor will not do this again
477  object_list.swap(object_list_);
478 
479  // Perform deletion in reverse order. We rely on it in functions which create
480  // a "lock" object - it is created as the first file and should be removed as
481  // last.
482  for (auto object_it = object_list.rbegin(); object_it != object_list.rend();
483  ++object_it) {
484  Status status = delete_fun_(std::move(object_it->first), object_it->second);
485  // Fail on first error. If the service is unavailable, every deletion
486  // would potentially keep retrying until the timeout passes - this would
487  // take way too much time and would be pointless.
488  if (!status.ok()) {
489  return status;
490  }
491  }
492  return Status();
493 }
494 
495 } // namespace internal
496 
497 } // namespace STORAGE_CLIENT_NS
498 } // namespace storage
499 } // namespace cloud
500 } // namespace google
google::cloud::storage::v1::ObjectReadStream
Defines a std::basic_istream<char> to read from a GCS Object.
Definition: object_stream.h:51
google::cloud::storage::v1::PolicyDocumentV4Result
Define a policy document result V4.
Definition: policy_document.h:192
google::cloud::storage::v1::Client
The Google Cloud Storage (GCS) Client.
Definition: client.h:187
google::cloud::storage::v1::CreateRandomPrefixName
std::string CreateRandomPrefixName(std::string const &prefix)
Create a random prefix for object names.
Definition: client.cc:436
service_account_credentials.h
google::cloud::storage::v1::ReadFromOffset
Download all the data from the GCS object starting at the given offset.
Definition: download_options.h:55
log.h
STORAGE_CLIENT_NS
#define STORAGE_CLIENT_NS
Definition: version.h:22
GCP_LOG
#define GCP_LOG(level)
client.h
google