GDAL
cpl_vsil_curl_class.h
1 /******************************************************************************
2  *
3  * Project: CPL - Common Portability Library
4  * Purpose: Declarations for /vsicurl/ and related file systems
5  * Author: Even Rouault, even.rouault at spatialys.com
6  *
7  ******************************************************************************
8  * Copyright (c) 2010-2018, Even Rouault <even.rouault at spatialys.com>
9  *
10  * Permission is hereby granted, free of charge, to any person obtaining a
11  * copy of this software and associated documentation files (the "Software"),
12  * to deal in the Software without restriction, including without limitation
13  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
14  * and/or sell copies of the Software, and to permit persons to whom the
15  * Software is furnished to do so, subject to the following conditions:
16  *
17  * The above copyright notice and this permission notice shall be included
18  * in all copies or substantial portions of the Software.
19  *
20  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
21  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
22  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
23  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
24  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
25  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
26  * DEALINGS IN THE SOFTWARE.
27  ****************************************************************************/
28 
29 #ifndef CPL_VSIL_CURL_CLASS_H_INCLUDED
30 #define CPL_VSIL_CURL_CLASS_H_INCLUDED
31 
32 #ifdef HAVE_CURL
33 
34 #include "cpl_aws.h"
35 #include "cpl_azure.h"
36 #include "cpl_port.h"
37 #include "cpl_json.h"
38 #include "cpl_string.h"
39 #include "cpl_vsil_curl_priv.h"
40 #include "cpl_mem_cache.h"
41 
42 #include "cpl_curl_priv.h"
43 
44 #include <set>
45 #include <map>
46 #include <memory>
47 #include <mutex>
48 
50 
51 // Leave it for backward compatibility, but deprecate.
52 #define HAVE_CURLINFO_REDIRECT_URL
53 
54 void VSICurlStreamingClearCache( void ); // from cpl_vsil_curl_streaming.cpp
55 
56 struct curl_slist* VSICurlSetOptions(CURL* hCurlHandle, const char* pszURL,
57  const char * const* papszOptions);
58 struct curl_slist* VSICurlMergeHeaders( struct curl_slist* poDest,
59  struct curl_slist* poSrcToDestroy );
60 
61 struct curl_slist* VSICurlSetContentTypeFromExt(struct curl_slist* polist,
62  const char *pszPath);
63 
64 struct curl_slist* VSICurlSetCreationHeadersFromOptions(struct curl_slist* headers,
65  CSLConstList papszOptions,
66  const char *pszPath);
67 
68 namespace cpl {
69 
70 typedef enum
71 {
72  EXIST_UNKNOWN = -1,
73  EXIST_NO,
74  EXIST_YES,
75 } ExistStatus;
76 
77 class FileProp
78 {
79  public:
80  unsigned int nGenerationAuthParameters = 0;
81  ExistStatus eExists = EXIST_UNKNOWN;
82  vsi_l_offset fileSize = 0;
83  time_t mTime = 0;
84  time_t nExpireTimestampLocal = 0;
85  CPLString osRedirectURL{};
86  bool bHasComputedFileSize = false;
87  bool bIsDirectory = false;
88  int nMode = 0; // st_mode member of struct stat
89  bool bS3LikeRedirect = false;
90  CPLString ETag{};
91 };
92 
93 struct CachedDirList
94 {
95  bool bGotFileList = false;
96  unsigned int nGenerationAuthParameters = 0;
97  CPLStringList oFileList{}; /* only file name without path */
98 };
99 
100 struct WriteFuncStruct
101 {
102  char* pBuffer = nullptr;
103  size_t nSize = 0;
104  bool bIsHTTP = false;
105  bool bMultiRange = false;
106  vsi_l_offset nStartOffset = 0;
107  vsi_l_offset nEndOffset = 0;
108  int nHTTPCode = 0;
109  vsi_l_offset nContentLength = 0;
110  bool bFoundContentRange = false;
111  bool bError = false;
112  bool bInterruptDownload = false;
113  bool bDetectRangeDownloadingError = false;
114  GIntBig nTimestampDate = 0; // Corresponds to Date: header field
115 
116  VSILFILE *fp = nullptr;
117  VSICurlReadCbkFunc pfnReadCbk = nullptr;
118  void *pReadCbkUserData = nullptr;
119  bool bInterrupted = false;
120 
121 #if !CURL_AT_LEAST_VERSION(7,54,0)
122  // Workaround to ignore extra HTTP response headers from
123  // proxies in older versions of curl.
124  // CURLOPT_SUPPRESS_CONNECT_HEADERS fixes this
125  bool bIsProxyConnectHeader = false;
126 #endif
127 };
128 
129 struct PutData
130 {
131  const GByte* pabyData = nullptr;
132  size_t nOff = 0;
133  size_t nTotalSize = 0;
134 
135  static size_t ReadCallBackBuffer( char *buffer, size_t size,
136  size_t nitems, void *instream )
137  {
138  PutData* poThis = static_cast<PutData *>(instream);
139  const size_t nSizeMax = size * nitems;
140  const size_t nSizeToWrite =
141  std::min(nSizeMax, poThis->nTotalSize - poThis->nOff);
142  memcpy(buffer, poThis->pabyData + poThis->nOff, nSizeToWrite);
143  poThis->nOff += nSizeToWrite;
144  return nSizeToWrite;
145  }
146 };
147 
148 /************************************************************************/
149 /* VSICurlFilesystemHandler */
150 /************************************************************************/
151 
152 class VSICurlHandle;
153 
154 class VSICurlFilesystemHandlerBase : public VSIFilesystemHandler
155 {
156  CPL_DISALLOW_COPY_ASSIGN(VSICurlFilesystemHandlerBase)
157 
158  struct FilenameOffsetPair
159  {
160  std::string filename_;
161  vsi_l_offset offset_;
162 
163  FilenameOffsetPair(const std::string& filename,
164  vsi_l_offset offset) :
165  filename_(filename), offset_(offset) {}
166 
167  bool operator==(const FilenameOffsetPair& other) const
168  {
169  return filename_ == other.filename_ &&
170  offset_ == other.offset_;
171  }
172  };
173  struct FilenameOffsetPairHasher
174  {
175  std::size_t operator()(const FilenameOffsetPair& k) const
176  {
177  return std::hash<std::string>()(k.filename_) ^
178  std::hash<vsi_l_offset>()(k.offset_);
179  }
180  };
181 
182  using RegionCacheType =
183  lru11::Cache<FilenameOffsetPair, std::shared_ptr<std::string>,
184  lru11::NullLock,
185  std::unordered_map<
186  FilenameOffsetPair,
187  typename std::list<lru11::KeyValuePair<FilenameOffsetPair,
188  std::shared_ptr<std::string>>>::iterator,
189  FilenameOffsetPairHasher>>;
190 
191  std::unique_ptr<RegionCacheType> m_poRegionCacheDoNotUseDirectly{}; // do not access directly. Use GetRegionCache();
192  RegionCacheType* GetRegionCache();
193 
194  lru11::Cache<std::string, FileProp> oCacheFileProp;
195 
196  int nCachedFilesInDirList = 0;
197  lru11::Cache<std::string, CachedDirList> oCacheDirList;
198 
199  char** ParseHTMLFileList(const char* pszFilename,
200  int nMaxFiles,
201  char* pszData,
202  bool* pbGotFileList);
203 
204 protected:
205  CPLMutex *hMutex = nullptr;
206 
207  virtual VSICurlHandle* CreateFileHandle(const char* pszFilename);
208  virtual char** GetFileList(const char *pszFilename,
209  int nMaxFiles,
210  bool* pbGotFileList);
211 
212  void RegisterEmptyDir( const CPLString& osDirname );
213 
214  bool AnalyseS3FileList( const CPLString& osBaseURL,
215  const char* pszXML,
216  CPLStringList& osFileList,
217  int nMaxFiles,
218  bool bIgnoreGlacierStorageClass,
219  bool& bIsTruncated );
220 
221  void AnalyseSwiftFileList( const CPLString& osBaseURL,
222  const CPLString& osPrefix,
223  const char* pszJson,
224  CPLStringList& osFileList,
225  int nMaxFilesThisQuery,
226  int nMaxFiles,
227  bool& bIsTruncated,
228  CPLString& osNextMarker );
229 
230  static const char* GetOptionsStatic();
231 
232  static bool IsAllowedFilename( const char* pszFilename );
233 
234  VSICurlFilesystemHandlerBase();
235 
236 public:
237  ~VSICurlFilesystemHandlerBase() override;
238 
239  VSIVirtualHandle *Open( const char *pszFilename,
240  const char *pszAccess,
241  bool bSetError,
242  CSLConstList /* papszOptions */ ) override;
243 
244  int Stat( const char *pszFilename, VSIStatBufL *pStatBuf,
245  int nFlags ) override;
246  int Unlink( const char *pszFilename ) override;
247  int Rename( const char *oldpath, const char *newpath ) override;
248  int Mkdir( const char *pszDirname, long nMode ) override;
249  int Rmdir( const char *pszDirname ) override;
250  char **ReadDir( const char *pszDirname ) override
251  { return ReadDirEx(pszDirname, 0); }
252  char **ReadDirEx( const char *pszDirname, int nMaxFiles ) override;
253  char **SiblingFiles( const char *pszFilename ) override;
254 
255  int HasOptimizedReadMultiRange( const char* /* pszPath */ )
256  override { return true; }
257 
258  const char* GetActualURL(const char* pszFilename) override;
259 
260  const char* GetOptions() override;
261 
262  char** GetFileMetadata( const char * pszFilename, const char* pszDomain,
263  CSLConstList papszOptions ) override;
264 
265  char **ReadDirInternal( const char *pszDirname, int nMaxFiles,
266  bool* pbGotFileList );
267  void InvalidateDirContent( const char *pszDirname );
268 
269  virtual const char* GetDebugKey() const = 0;
270 
271  virtual CPLString GetFSPrefix() const = 0;
272  virtual bool AllowCachedDataFor(const char* pszFilename);
273 
274  std::shared_ptr<std::string> GetRegion( const char* pszURL,
275  vsi_l_offset nFileOffsetStart );
276 
277  void AddRegion( const char* pszURL,
278  vsi_l_offset nFileOffsetStart,
279  size_t nSize,
280  const char *pData );
281 
282  bool GetCachedFileProp( const char* pszURL,
283  FileProp& oFileProp );
284  void SetCachedFileProp( const char* pszURL,
285  FileProp& oFileProp );
286  void InvalidateCachedData( const char* pszURL );
287 
288  CURLM *GetCurlMultiHandleFor( const CPLString& osURL );
289 
290  virtual void ClearCache();
291  virtual void PartialClearCache(const char* pszFilename);
292 
293 
294  bool GetCachedDirList( const char* pszURL,
295  CachedDirList& oCachedDirList );
296  void SetCachedDirList( const char* pszURL,
297  CachedDirList& oCachedDirList );
298  bool ExistsInCacheDirList( const CPLString& osDirname, bool *pbIsDir );
299 
300  virtual CPLString GetURLFromFilename( const CPLString& osFilename );
301 
302  std::string GetStreamingFilename(const std::string& osFilename) const override = 0;
303 };
304 
305 
306 class VSICurlFilesystemHandler: public VSICurlFilesystemHandlerBase
307 {
308  CPL_DISALLOW_COPY_ASSIGN(VSICurlFilesystemHandler)
309 
310 public:
311  VSICurlFilesystemHandler() = default;
312 
313  const char* GetDebugKey() const override { return "VSICURL"; }
314 
315  CPLString GetFSPrefix() const override { return "/vsicurl/"; }
316 
317  std::string GetStreamingFilename(const std::string& osFilename) const override;
318 };
319 
320 /************************************************************************/
321 /* VSICurlHandle */
322 /************************************************************************/
323 
324 class VSICurlHandle : public VSIVirtualHandle
325 {
326  CPL_DISALLOW_COPY_ASSIGN(VSICurlHandle)
327 
328  protected:
329  VSICurlFilesystemHandlerBase* poFS = nullptr;
330 
331  bool m_bCached = true;
332 
333  FileProp oFileProp{};
334 
335  CPLString m_osFilename{}; // e.g "/vsicurl/http://example.com/foo"
336  char* m_pszURL = nullptr; // e.g "http://example.com/foo"
337  std::string m_osQueryString{}; // e.g. an Azure SAS
338 
339  char **m_papszHTTPOptions = nullptr;
340 
341  vsi_l_offset lastDownloadedOffset = VSI_L_OFFSET_MAX;
342  int nBlocksToDownload = 1;
343 
344  bool bStopOnInterruptUntilUninstall = false;
345  bool bInterrupted = false;
346  VSICurlReadCbkFunc pfnReadCbk = nullptr;
347  void *pReadCbkUserData = nullptr;
348 
349  int m_nMaxRetry = 0;
350  double m_dfRetryDelay = 0.0;
351 
352  CPLStringList m_aosHeaders{};
353 
354  void DownloadRegionPostProcess( const vsi_l_offset startOffset,
355  const int nBlocks,
356  const char* pBuffer,
357  size_t nSize );
358 
359  private:
360 
361  vsi_l_offset curOffset = 0;
362 
363  bool bEOF = false;
364 
365  virtual std::string DownloadRegion(vsi_l_offset startOffset, int nBlocks);
366 
367  bool m_bUseHead = false;
368  bool m_bUseRedirectURLIfNoQueryStringParams = false;
369 
370  int ReadMultiRangeSingleGet( int nRanges, void ** ppData,
371  const vsi_l_offset* panOffsets,
372  const size_t* panSizes );
373  CPLString GetRedirectURLIfValid(bool& bHasExpired);
374 
375  protected:
376  virtual struct curl_slist* GetCurlHeaders( const CPLString& /*osVerb*/,
377  const struct curl_slist* /* psExistingHeaders */)
378  { return nullptr; }
379  virtual bool AllowAutomaticRedirection() { return true; }
380  virtual bool CanRestartOnError( const char*, const char*, bool ) { return false; }
381  virtual bool UseLimitRangeGetInsteadOfHead() { return false; }
382  virtual bool IsDirectoryFromExists( const char* /*pszVerb*/, int /*response_code*/ ) { return false; }
383  virtual void ProcessGetFileSizeResult(const char* /* pszContent */ ) {}
384  void SetURL(const char* pszURL);
385  virtual bool Authenticate() { return false; }
386 
387  public:
388 
389  VSICurlHandle( VSICurlFilesystemHandlerBase* poFS,
390  const char* pszFilename,
391  const char* pszURLIn = nullptr );
392  ~VSICurlHandle() override;
393 
394  int Seek( vsi_l_offset nOffset, int nWhence ) override;
395  vsi_l_offset Tell() override;
396  size_t Read( void *pBuffer, size_t nSize, size_t nMemb ) override;
397  int ReadMultiRange( int nRanges, void ** ppData,
398  const vsi_l_offset* panOffsets,
399  const size_t* panSizes ) override;
400  size_t Write( const void *pBuffer, size_t nSize, size_t nMemb ) override;
401  int Eof() override;
402  int Flush() override;
403  int Close() override;
404 
405  bool IsKnownFileSize() const { return oFileProp.bHasComputedFileSize; }
406  vsi_l_offset GetFileSizeOrHeaders(bool bSetError, bool bGetHeaders);
407  virtual vsi_l_offset GetFileSize( bool bSetError ) { return GetFileSizeOrHeaders(bSetError, false); }
408  bool Exists( bool bSetError );
409  bool IsDirectory() const { return oFileProp.bIsDirectory; }
410  int GetMode() const { return oFileProp.nMode; }
411  time_t GetMTime() const { return oFileProp.mTime; }
412  const CPLStringList& GetHeaders() { return m_aosHeaders; }
413 
414  int InstallReadCbk( VSICurlReadCbkFunc pfnReadCbk,
415  void* pfnUserData,
416  int bStopOnInterruptUntilUninstall );
417  int UninstallReadCbk();
418 
419  const char *GetURL() const { return m_pszURL; }
420 };
421 
422 /************************************************************************/
423 /* IVSIS3LikeFSHandler */
424 /************************************************************************/
425 
426 class IVSIS3LikeFSHandler: public VSICurlFilesystemHandlerBase
427 {
428  CPL_DISALLOW_COPY_ASSIGN(IVSIS3LikeFSHandler)
429 
430  bool CopyFile(VSILFILE* fpIn,
431  vsi_l_offset nSourceSize,
432  const char* pszSource,
433  const char* pszTarget,
434  GDALProgressFunc pProgressFunc,
435  void *pProgressData);
436  virtual int MkdirInternal( const char *pszDirname, long nMode, bool bDoStatCheck );
437 
438  protected:
439  char** GetFileList( const char *pszFilename,
440  int nMaxFiles,
441  bool* pbGotFileList ) override;
442 
443  virtual IVSIS3LikeHandleHelper* CreateHandleHelper(
444  const char* pszURI, bool bAllowNoObject) = 0;
445 
446  virtual int CopyObject( const char *oldpath, const char *newpath,
447  CSLConstList papszMetadata );
448 
449  int RmdirRecursiveInternal( const char* pszDirname, int nBatchSize);
450 
451  IVSIS3LikeFSHandler() = default;
452 
453  public:
454  int Unlink( const char *pszFilename ) override;
455  int Mkdir( const char *pszDirname, long nMode ) override;
456  int Rmdir( const char *pszDirname ) override;
457  int Stat( const char *pszFilename, VSIStatBufL *pStatBuf,
458  int nFlags ) override;
459  int Rename( const char *oldpath, const char *newpath ) override;
460 
461  virtual int DeleteObject( const char *pszFilename );
462 
463  virtual void UpdateMapFromHandle(IVSIS3LikeHandleHelper*) {}
464  virtual void UpdateHandleFromMap( IVSIS3LikeHandleHelper * ) {}
465 
466  bool Sync( const char* pszSource, const char* pszTarget,
467  const char* const * papszOptions,
468  GDALProgressFunc pProgressFunc,
469  void *pProgressData,
470  char*** ppapszOutputs ) override;
471 
472  VSIDIR* OpenDir( const char *pszPath, int nRecurseDepth,
473  const char* const *papszOptions) override;
474 
475  // Multipart upload
476  virtual bool SupportsParallelMultipartUpload() const { return false; }
477 
478  virtual CPLString InitiateMultipartUpload(
479  const std::string& osFilename,
480  IVSIS3LikeHandleHelper *poS3HandleHelper,
481  int nMaxRetry,
482  double dfRetryDelay,
483  CSLConstList papszOptions);
484  virtual CPLString UploadPart(const CPLString& osFilename,
485  int nPartNumber,
486  const std::string& osUploadID,
487  vsi_l_offset nPosition,
488  const void* pabyBuffer,
489  size_t nBufferSize,
490  IVSIS3LikeHandleHelper *poS3HandleHelper,
491  int nMaxRetry,
492  double dfRetryDelay);
493  virtual bool CompleteMultipart(const CPLString& osFilename,
494  const CPLString& osUploadID,
495  const std::vector<CPLString>& aosEtags,
496  vsi_l_offset nTotalSize,
497  IVSIS3LikeHandleHelper *poS3HandleHelper,
498  int nMaxRetry,
499  double dfRetryDelay);
500  virtual bool AbortMultipart(const CPLString& osFilename,
501  const CPLString& osUploadID,
502  IVSIS3LikeHandleHelper *poS3HandleHelper,
503  int nMaxRetry,
504  double dfRetryDelay);
505 
506  bool AbortPendingUploads(const char* pszFilename) override;
507 };
508 
509 /************************************************************************/
510 /* IVSIS3LikeHandle */
511 /************************************************************************/
512 
513 class IVSIS3LikeHandle: public VSICurlHandle
514 {
515  CPL_DISALLOW_COPY_ASSIGN(IVSIS3LikeHandle)
516 
517  protected:
518  bool UseLimitRangeGetInsteadOfHead() override { return true; }
519  bool IsDirectoryFromExists( const char* pszVerb,
520  int response_code ) override
521  {
522  // A bit dirty, but on S3, a GET on a existing directory returns a 416
523  return response_code == 416 && EQUAL(pszVerb, "GET") &&
524  CPLString(m_pszURL).back() == '/';
525  }
526  void ProcessGetFileSizeResult( const char* pszContent ) override
527  {
528  oFileProp.bIsDirectory = strstr(pszContent, "ListBucketResult") != nullptr;
529  }
530 
531  public:
532  IVSIS3LikeHandle( VSICurlFilesystemHandlerBase* poFSIn,
533  const char* pszFilename,
534  const char* pszURLIn ) :
535  VSICurlHandle(poFSIn, pszFilename, pszURLIn) {}
536  ~IVSIS3LikeHandle() override {}
537 };
538 
539 /************************************************************************/
540 /* VSIS3WriteHandle */
541 /************************************************************************/
542 
543 class VSIS3WriteHandle final : public VSIVirtualHandle
544 {
545  CPL_DISALLOW_COPY_ASSIGN(VSIS3WriteHandle)
546 
547  IVSIS3LikeFSHandler *m_poFS = nullptr;
548  CPLString m_osFilename{};
549  IVSIS3LikeHandleHelper *m_poS3HandleHelper = nullptr;
550  bool m_bUseChunked = false;
551  CPLStringList m_aosOptions{};
552 
553  vsi_l_offset m_nCurOffset = 0;
554  int m_nBufferOff = 0;
555  int m_nBufferSize = 0;
556  bool m_bClosed = false;
557  GByte *m_pabyBuffer = nullptr;
558  CPLString m_osUploadID{};
559  int m_nPartNumber = 0;
560  std::vector<CPLString> m_aosEtags{};
561  bool m_bError = false;
562 
563  CURLM *m_hCurlMulti = nullptr;
564  CURL *m_hCurl = nullptr;
565  const void *m_pBuffer = nullptr;
566  CPLString m_osCurlErrBuf{};
567  size_t m_nChunkedBufferOff = 0;
568  size_t m_nChunkedBufferSize = 0;
569  size_t m_nWrittenInPUT = 0;
570 
571  int m_nMaxRetry = 0;
572  double m_dfRetryDelay = 0.0;
573  WriteFuncStruct m_sWriteFuncHeaderData{};
574 
575  bool UploadPart();
576  bool DoSinglePartPUT();
577 
578  static size_t ReadCallBackBufferChunked( char *buffer, size_t size,
579  size_t nitems, void *instream );
580  size_t WriteChunked( const void *pBuffer,
581  size_t nSize, size_t nMemb );
582  int FinishChunkedTransfer();
583 
584  void InvalidateParentDirectory();
585 
586  public:
587  VSIS3WriteHandle( IVSIS3LikeFSHandler* poFS,
588  const char* pszFilename,
589  IVSIS3LikeHandleHelper* poS3HandleHelper,
590  bool bUseChunked,
591  CSLConstList papszOptions );
592  ~VSIS3WriteHandle() override;
593 
594  int Seek( vsi_l_offset nOffset, int nWhence ) override;
595  vsi_l_offset Tell() override;
596  size_t Read( void *pBuffer, size_t nSize, size_t nMemb ) override;
597  size_t Write( const void *pBuffer, size_t nSize, size_t nMemb ) override;
598  int Eof() override;
599  int Close() override;
600 
601  bool IsOK() { return m_bUseChunked || m_pabyBuffer != nullptr; }
602 };
603 
604 /************************************************************************/
605 /* VSIAppendWriteHandle */
606 /************************************************************************/
607 
608 class VSIAppendWriteHandle : public VSIVirtualHandle
609 {
610  CPL_DISALLOW_COPY_ASSIGN(VSIAppendWriteHandle)
611 
612  protected:
613 
614  VSICurlFilesystemHandlerBase* m_poFS = nullptr;
615  CPLString m_osFSPrefix{};
616  CPLString m_osFilename{};
617 
618  vsi_l_offset m_nCurOffset = 0;
619  int m_nBufferOff = 0;
620  int m_nBufferSize = 0;
621  int m_nBufferOffReadCallback = 0;
622  bool m_bClosed = false;
623  GByte *m_pabyBuffer = nullptr;
624  bool m_bError = false;
625 
626  static size_t ReadCallBackBuffer( char *buffer, size_t size,
627  size_t nitems, void *instream );
628  virtual bool Send(bool bIsLastBlock) = 0;
629 
630  public:
631  VSIAppendWriteHandle( VSICurlFilesystemHandlerBase* poFS,
632  const char* pszFSPrefix,
633  const char* pszFilename,
634  int nChunkSize );
635  virtual ~VSIAppendWriteHandle();
636 
637  int Seek( vsi_l_offset nOffset, int nWhence ) override;
638  vsi_l_offset Tell() override;
639  size_t Read( void *pBuffer, size_t nSize, size_t nMemb ) override;
640  size_t Write( const void *pBuffer, size_t nSize, size_t nMemb ) override;
641  int Eof() override;
642  int Close() override;
643 
644  bool IsOK() { return m_pabyBuffer != nullptr; }
645 };
646 
647 /************************************************************************/
648 /* CurlRequestHelper */
649 /************************************************************************/
650 
651 struct CurlRequestHelper
652 {
653  WriteFuncStruct sWriteFuncData{};
654  WriteFuncStruct sWriteFuncHeaderData{};
655  char szCurlErrBuf[CURL_ERROR_SIZE+1] = {};
656 
657  CurlRequestHelper();
658  ~CurlRequestHelper();
659  long perform(CURL* hCurlHandle,
660  struct curl_slist* headers, // ownership transferred
661  VSICurlFilesystemHandlerBase *poFS,
662  IVSIS3LikeHandleHelper *poS3HandleHelper);
663 };
664 
665 /************************************************************************/
666 /* NetworkStatisticsLogger */
667 /************************************************************************/
668 
669 class NetworkStatisticsLogger
670 {
671  static int gnEnabled;
672  static NetworkStatisticsLogger gInstance;
673 
674  NetworkStatisticsLogger() = default;
675 
676  std::mutex m_mutex{};
677 
678  struct Counters
679  {
680  GIntBig nHEAD = 0;
681  GIntBig nGET = 0;
682  GIntBig nPUT = 0;
683  GIntBig nPOST = 0;
684  GIntBig nDELETE = 0;
685  GIntBig nGETDownloadedBytes = 0;
686  GIntBig nPUTUploadedBytes = 0;
687  GIntBig nPOSTDownloadedBytes = 0;
688  GIntBig nPOSTUploadedBytes = 0;
689  };
690 
691  enum class ContextPathType
692  {
693  FILESYSTEM,
694  FILE,
695  ACTION,
696  };
697 
698  struct ContextPathItem
699  {
700  ContextPathType eType;
701  CPLString osName;
702 
703  ContextPathItem(ContextPathType eTypeIn, const CPLString& osNameIn):
704  eType(eTypeIn), osName(osNameIn) {}
705 
706  bool operator< (const ContextPathItem& other ) const
707  {
708  if( static_cast<int>(eType) < static_cast<int>(other.eType) )
709  return true;
710  if( static_cast<int>(eType) > static_cast<int>(other.eType) )
711  return false;
712  return osName < other.osName;
713  }
714  };
715 
716  struct Stats
717  {
718  Counters counters{};
719  std::map<ContextPathItem, Stats> children{};
720 
721  void AsJSON(CPLJSONObject& oJSON) const;
722  };
723 
724  // Workaround bug in Coverity Scan
725  // coverity[generated_default_constructor_used_in_field_initializer]
726  Stats m_stats{};
727  std::map<GIntBig, std::vector<ContextPathItem>> m_mapThreadIdToContextPath{};
728 
729  static void ReadEnabled();
730 
731  std::vector<Counters*> GetCountersForContext();
732 
733 public:
734 
735  static inline bool IsEnabled()
736  {
737  if( gnEnabled < 0)
738  {
739  ReadEnabled();
740  }
741  return gnEnabled == TRUE;
742  }
743 
744  static void EnterFileSystem(const char* pszName);
745 
746  static void LeaveFileSystem();
747 
748  static void EnterFile(const char* pszName);
749 
750  static void LeaveFile();
751 
752  static void EnterAction(const char* pszName);
753 
754  static void LeaveAction();
755 
756  static void LogHEAD();
757 
758  static void LogGET(size_t nDownloadedBytes);
759 
760  static void LogPUT(size_t nUploadedBytes);
761 
762  static void LogPOST(size_t nUploadedBytes,
763  size_t nDownloadedBytes);
764 
765  static void LogDELETE();
766 
767  static void Reset();
768 
769  static CPLString GetReportAsSerializedJSON();
770 };
771 
772 struct NetworkStatisticsFileSystem
773 {
774  inline explicit NetworkStatisticsFileSystem(const char* pszName) {
775  NetworkStatisticsLogger::EnterFileSystem(pszName);
776  }
777 
778  inline ~NetworkStatisticsFileSystem()
779  {
780  NetworkStatisticsLogger::LeaveFileSystem();
781  }
782 };
783 
784 struct NetworkStatisticsFile
785 {
786  inline explicit NetworkStatisticsFile(const char* pszName) {
787  NetworkStatisticsLogger::EnterFile(pszName);
788  }
789 
790  inline ~NetworkStatisticsFile()
791  {
792  NetworkStatisticsLogger::LeaveFile();
793  }
794 };
795 
796 struct NetworkStatisticsAction
797 {
798  inline explicit NetworkStatisticsAction(const char* pszName) {
799  NetworkStatisticsLogger::EnterAction(pszName);
800  }
801 
802  inline ~NetworkStatisticsAction()
803  {
804  NetworkStatisticsLogger::LeaveAction();
805  }
806 };
807 
808 
809 int VSICURLGetDownloadChunkSize();
810 
811 void VSICURLInitWriteFuncStruct( WriteFuncStruct *psStruct,
812  VSILFILE *fp,
813  VSICurlReadCbkFunc pfnReadCbk,
814  void *pReadCbkUserData );
815 size_t VSICurlHandleWriteFunc( void *buffer, size_t count,
816  size_t nmemb, void *req );
817 void MultiPerform(CURLM* hCurlMultiHandle,
818  CURL* hEasyHandle = nullptr);
819 void VSICURLResetHeaderAndWriterFunctions(CURL* hCurlHandle);
820 
821 int VSICurlParseUnixPermissions(const char* pszPermissions);
822 
823 } // namespace cpl
824 
826 
827 #endif // HAVE_CURL
828 
829 #endif // CPL_VSIL_CURL_CLASS_H_INCLUDED
The CPLJSONArray class holds JSON object from CPLJSONDocument.
Definition: cpl_json.h:54
String list class designed around our use of C "char**" string lists.
Definition: cpl_string.h:442
Convenient string class based on std::string.
Definition: cpl_string.h:333
Virtual file handle.
Definition: cpl_vsi_virtual.h:56
Interface for read and write JSON documents.
Core portability definitions for CPL.
#define EQUAL(a, b)
Alias for strcasecmp() == 0.
Definition: cpl_port.h:568
#define CPL_DISALLOW_COPY_ASSIGN(ClassName)
Helper to remove the copy and assignment constructors so that the compiler will not generate the defa...
Definition: cpl_port.h:955
char ** CSLConstList
Type of a constant null-terminated list of nul terminated strings.
Definition: cpl_port.h:1169
unsigned char GByte
Unsigned byte type.
Definition: cpl_port.h:215
long long GIntBig
Large signed integer type (generally 64-bit integer type).
Definition: cpl_port.h:244
Various convenience functions for working with strings and string lists.
#define VSI_L_OFFSET_MAX
Maximum value for a file offset.
Definition: cpl_vsi.h:142
struct VSIDIR VSIDIR
Opaque type for a directory iterator.
Definition: cpl_vsi.h:323
struct VSI_STAT64_T VSIStatBufL
Type for VSIStatL()
Definition: cpl_vsi.h:195
FILE VSILFILE
Opaque type for a FILE that implements the VSIVirtualHandle API.
Definition: cpl_vsi.h:156
GUIntBig vsi_l_offset
Type for a file offset.
Definition: cpl_vsi.h:140