29 #ifndef CPL_VSIL_CURL_CLASS_H_INCLUDED
30 #define CPL_VSIL_CURL_CLASS_H_INCLUDED
35 #include "cpl_azure.h"
39 #include "cpl_vsil_curl_priv.h"
40 #include "cpl_mem_cache.h"
42 #include "cpl_curl_priv.h"
52 #define HAVE_CURLINFO_REDIRECT_URL
54 void VSICurlStreamingClearCache(
void );
56 struct curl_slist* VSICurlSetOptions(CURL* hCurlHandle,
const char* pszURL,
57 const char *
const* papszOptions);
58 struct curl_slist* VSICurlMergeHeaders(
struct curl_slist* poDest,
59 struct curl_slist* poSrcToDestroy );
61 struct curl_slist* VSICurlSetContentTypeFromExt(
struct curl_slist* polist,
64 struct curl_slist* VSICurlSetCreationHeadersFromOptions(
struct curl_slist* headers,
80 unsigned int nGenerationAuthParameters = 0;
81 ExistStatus eExists = EXIST_UNKNOWN;
84 time_t nExpireTimestampLocal = 0;
86 bool bHasComputedFileSize =
false;
87 bool bIsDirectory =
false;
89 bool bS3LikeRedirect =
false;
95 bool bGotFileList =
false;
96 unsigned int nGenerationAuthParameters = 0;
100 struct WriteFuncStruct
102 char* pBuffer =
nullptr;
104 bool bIsHTTP =
false;
105 bool bMultiRange =
false;
110 bool bFoundContentRange =
false;
112 bool bInterruptDownload =
false;
113 bool bDetectRangeDownloadingError =
false;
117 VSICurlReadCbkFunc pfnReadCbk =
nullptr;
118 void *pReadCbkUserData =
nullptr;
119 bool bInterrupted =
false;
121 #if !CURL_AT_LEAST_VERSION(7,54,0)
125 bool bIsProxyConnectHeader =
false;
131 const GByte* pabyData =
nullptr;
133 size_t nTotalSize = 0;
135 static size_t ReadCallBackBuffer(
char *buffer,
size_t size,
136 size_t nitems,
void *instream )
138 PutData* poThis =
static_cast<PutData *
>(instream);
139 const size_t nSizeMax = size * nitems;
140 const size_t nSizeToWrite =
141 std::min(nSizeMax, poThis->nTotalSize - poThis->nOff);
142 memcpy(buffer, poThis->pabyData + poThis->nOff, nSizeToWrite);
143 poThis->nOff += nSizeToWrite;
154 class VSICurlFilesystemHandlerBase :
public VSIFilesystemHandler
158 struct FilenameOffsetPair
160 std::string filename_;
163 FilenameOffsetPair(
const std::string& filename,
165 filename_(filename), offset_(offset) {}
167 bool operator==(
const FilenameOffsetPair& other)
const
169 return filename_ == other.filename_ &&
170 offset_ == other.offset_;
173 struct FilenameOffsetPairHasher
175 std::size_t operator()(
const FilenameOffsetPair& k)
const
177 return std::hash<std::string>()(k.filename_) ^
178 std::hash<vsi_l_offset>()(k.offset_);
182 using RegionCacheType =
183 lru11::Cache<FilenameOffsetPair, std::shared_ptr<std::string>,
187 typename std::list<lru11::KeyValuePair<FilenameOffsetPair,
188 std::shared_ptr<std::string>>>::iterator,
189 FilenameOffsetPairHasher>>;
191 std::unique_ptr<RegionCacheType> m_poRegionCacheDoNotUseDirectly{};
192 RegionCacheType* GetRegionCache();
194 lru11::Cache<std::string, FileProp> oCacheFileProp;
196 int nCachedFilesInDirList = 0;
197 lru11::Cache<std::string, CachedDirList> oCacheDirList;
199 char** ParseHTMLFileList(
const char* pszFilename,
202 bool* pbGotFileList);
205 CPLMutex *hMutex =
nullptr;
207 virtual VSICurlHandle* CreateFileHandle(
const char* pszFilename);
208 virtual char** GetFileList(
const char *pszFilename,
210 bool* pbGotFileList);
212 void RegisterEmptyDir(
const CPLString& osDirname );
214 bool AnalyseS3FileList(
const CPLString& osBaseURL,
218 bool bIgnoreGlacierStorageClass,
219 bool& bIsTruncated );
221 void AnalyseSwiftFileList(
const CPLString& osBaseURL,
225 int nMaxFilesThisQuery,
230 static const char* GetOptionsStatic();
232 static bool IsAllowedFilename(
const char* pszFilename );
234 VSICurlFilesystemHandlerBase();
237 ~VSICurlFilesystemHandlerBase()
override;
240 const char *pszAccess,
244 int Stat(
const char *pszFilename,
VSIStatBufL *pStatBuf,
245 int nFlags )
override;
246 int Unlink(
const char *pszFilename )
override;
247 int Rename(
const char *oldpath,
const char *newpath )
override;
248 int Mkdir(
const char *pszDirname,
long nMode )
override;
249 int Rmdir(
const char *pszDirname )
override;
250 char **ReadDir(
const char *pszDirname )
override
251 {
return ReadDirEx(pszDirname, 0); }
252 char **ReadDirEx(
const char *pszDirname,
int nMaxFiles )
override;
253 char **SiblingFiles(
const char *pszFilename )
override;
255 int HasOptimizedReadMultiRange(
const char* )
256 override {
return true; }
258 const char* GetActualURL(
const char* pszFilename)
override;
260 const char* GetOptions()
override;
262 char** GetFileMetadata(
const char * pszFilename,
const char* pszDomain,
265 char **ReadDirInternal(
const char *pszDirname,
int nMaxFiles,
266 bool* pbGotFileList );
267 void InvalidateDirContent(
const char *pszDirname );
269 virtual const char* GetDebugKey()
const = 0;
271 virtual CPLString GetFSPrefix()
const = 0;
272 virtual bool AllowCachedDataFor(
const char* pszFilename);
274 std::shared_ptr<std::string> GetRegion(
const char* pszURL,
277 void AddRegion(
const char* pszURL,
282 bool GetCachedFileProp(
const char* pszURL,
283 FileProp& oFileProp );
284 void SetCachedFileProp(
const char* pszURL,
285 FileProp& oFileProp );
286 void InvalidateCachedData(
const char* pszURL );
288 CURLM *GetCurlMultiHandleFor(
const CPLString& osURL );
290 virtual void ClearCache();
291 virtual void PartialClearCache(
const char* pszFilename);
294 bool GetCachedDirList(
const char* pszURL,
295 CachedDirList& oCachedDirList );
296 void SetCachedDirList(
const char* pszURL,
297 CachedDirList& oCachedDirList );
298 bool ExistsInCacheDirList(
const CPLString& osDirname,
bool *pbIsDir );
302 std::string GetStreamingFilename(
const std::string& osFilename)
const override = 0;
306 class VSICurlFilesystemHandler:
public VSICurlFilesystemHandlerBase
311 VSICurlFilesystemHandler() =
default;
313 const char* GetDebugKey()
const override {
return "VSICURL"; }
315 CPLString GetFSPrefix()
const override {
return "/vsicurl/"; }
317 std::string GetStreamingFilename(
const std::string& osFilename)
const override;
329 VSICurlFilesystemHandlerBase* poFS =
nullptr;
331 bool m_bCached =
true;
333 FileProp oFileProp{};
336 char* m_pszURL =
nullptr;
337 std::string m_osQueryString{};
339 char **m_papszHTTPOptions =
nullptr;
342 int nBlocksToDownload = 1;
344 bool bStopOnInterruptUntilUninstall =
false;
345 bool bInterrupted =
false;
346 VSICurlReadCbkFunc pfnReadCbk =
nullptr;
347 void *pReadCbkUserData =
nullptr;
350 double m_dfRetryDelay = 0.0;
354 void DownloadRegionPostProcess(
const vsi_l_offset startOffset,
365 virtual std::string DownloadRegion(
vsi_l_offset startOffset,
int nBlocks);
367 bool m_bUseHead =
false;
368 bool m_bUseRedirectURLIfNoQueryStringParams =
false;
370 int ReadMultiRangeSingleGet(
int nRanges,
void ** ppData,
372 const size_t* panSizes );
373 CPLString GetRedirectURLIfValid(
bool& bHasExpired);
376 virtual struct curl_slist* GetCurlHeaders(
const CPLString& ,
377 const struct curl_slist* )
379 virtual bool AllowAutomaticRedirection() {
return true; }
380 virtual bool CanRestartOnError(
const char*,
const char*,
bool ) {
return false; }
381 virtual bool UseLimitRangeGetInsteadOfHead() {
return false; }
382 virtual bool IsDirectoryFromExists(
const char* ,
int ) {
return false; }
383 virtual void ProcessGetFileSizeResult(
const char* ) {}
384 void SetURL(
const char* pszURL);
385 virtual bool Authenticate() {
return false; }
389 VSICurlHandle( VSICurlFilesystemHandlerBase* poFS,
390 const char* pszFilename,
391 const char* pszURLIn =
nullptr );
392 ~VSICurlHandle()
override;
396 size_t Read(
void *pBuffer,
size_t nSize,
size_t nMemb )
override;
397 int ReadMultiRange(
int nRanges,
void ** ppData,
399 const size_t* panSizes )
override;
400 size_t Write(
const void *pBuffer,
size_t nSize,
size_t nMemb )
override;
402 int Flush()
override;
403 int Close()
override;
405 bool IsKnownFileSize()
const {
return oFileProp.bHasComputedFileSize; }
406 vsi_l_offset GetFileSizeOrHeaders(
bool bSetError,
bool bGetHeaders);
407 virtual vsi_l_offset GetFileSize(
bool bSetError ) {
return GetFileSizeOrHeaders(bSetError,
false); }
408 bool Exists(
bool bSetError );
409 bool IsDirectory()
const {
return oFileProp.bIsDirectory; }
410 int GetMode()
const {
return oFileProp.nMode; }
411 time_t GetMTime()
const {
return oFileProp.mTime; }
414 int InstallReadCbk( VSICurlReadCbkFunc pfnReadCbk,
416 int bStopOnInterruptUntilUninstall );
417 int UninstallReadCbk();
419 const char *GetURL()
const {
return m_pszURL; }
426 class IVSIS3LikeFSHandler:
public VSICurlFilesystemHandlerBase
432 const char* pszSource,
433 const char* pszTarget,
434 GDALProgressFunc pProgressFunc,
435 void *pProgressData);
436 virtual int MkdirInternal(
const char *pszDirname,
long nMode,
bool bDoStatCheck );
439 char** GetFileList(
const char *pszFilename,
441 bool* pbGotFileList )
override;
443 virtual IVSIS3LikeHandleHelper* CreateHandleHelper(
444 const char* pszURI,
bool bAllowNoObject) = 0;
446 virtual int CopyObject(
const char *oldpath,
const char *newpath,
449 int RmdirRecursiveInternal(
const char* pszDirname,
int nBatchSize);
451 IVSIS3LikeFSHandler() =
default;
454 int Unlink(
const char *pszFilename )
override;
455 int Mkdir(
const char *pszDirname,
long nMode )
override;
456 int Rmdir(
const char *pszDirname )
override;
457 int Stat(
const char *pszFilename,
VSIStatBufL *pStatBuf,
458 int nFlags )
override;
459 int Rename(
const char *oldpath,
const char *newpath )
override;
461 virtual int DeleteObject(
const char *pszFilename );
463 virtual void UpdateMapFromHandle(IVSIS3LikeHandleHelper*) {}
464 virtual void UpdateHandleFromMap( IVSIS3LikeHandleHelper * ) {}
466 bool Sync(
const char* pszSource,
const char* pszTarget,
467 const char*
const * papszOptions,
468 GDALProgressFunc pProgressFunc,
470 char*** ppapszOutputs )
override;
472 VSIDIR* OpenDir(
const char *pszPath,
int nRecurseDepth,
473 const char*
const *papszOptions)
override;
476 virtual bool SupportsParallelMultipartUpload()
const {
return false; }
478 virtual CPLString InitiateMultipartUpload(
479 const std::string& osFilename,
480 IVSIS3LikeHandleHelper *poS3HandleHelper,
486 const std::string& osUploadID,
488 const void* pabyBuffer,
490 IVSIS3LikeHandleHelper *poS3HandleHelper,
492 double dfRetryDelay);
493 virtual bool CompleteMultipart(
const CPLString& osFilename,
495 const std::vector<CPLString>& aosEtags,
497 IVSIS3LikeHandleHelper *poS3HandleHelper,
499 double dfRetryDelay);
500 virtual bool AbortMultipart(
const CPLString& osFilename,
502 IVSIS3LikeHandleHelper *poS3HandleHelper,
504 double dfRetryDelay);
506 bool AbortPendingUploads(
const char* pszFilename)
override;
513 class IVSIS3LikeHandle:
public VSICurlHandle
518 bool UseLimitRangeGetInsteadOfHead()
override {
return true; }
519 bool IsDirectoryFromExists(
const char* pszVerb,
520 int response_code )
override
523 return response_code == 416 &&
EQUAL(pszVerb,
"GET") &&
526 void ProcessGetFileSizeResult(
const char* pszContent )
override
528 oFileProp.bIsDirectory = strstr(pszContent,
"ListBucketResult") !=
nullptr;
532 IVSIS3LikeHandle( VSICurlFilesystemHandlerBase* poFSIn,
533 const char* pszFilename,
534 const char* pszURLIn ) :
535 VSICurlHandle(poFSIn, pszFilename, pszURLIn) {}
536 ~IVSIS3LikeHandle()
override {}
547 IVSIS3LikeFSHandler *m_poFS =
nullptr;
549 IVSIS3LikeHandleHelper *m_poS3HandleHelper =
nullptr;
550 bool m_bUseChunked =
false;
554 int m_nBufferOff = 0;
555 int m_nBufferSize = 0;
556 bool m_bClosed =
false;
557 GByte *m_pabyBuffer =
nullptr;
559 int m_nPartNumber = 0;
560 std::vector<CPLString> m_aosEtags{};
561 bool m_bError =
false;
563 CURLM *m_hCurlMulti =
nullptr;
564 CURL *m_hCurl =
nullptr;
565 const void *m_pBuffer =
nullptr;
567 size_t m_nChunkedBufferOff = 0;
568 size_t m_nChunkedBufferSize = 0;
569 size_t m_nWrittenInPUT = 0;
572 double m_dfRetryDelay = 0.0;
573 WriteFuncStruct m_sWriteFuncHeaderData{};
576 bool DoSinglePartPUT();
578 static size_t ReadCallBackBufferChunked(
char *buffer,
size_t size,
579 size_t nitems,
void *instream );
580 size_t WriteChunked(
const void *pBuffer,
581 size_t nSize,
size_t nMemb );
582 int FinishChunkedTransfer();
584 void InvalidateParentDirectory();
587 VSIS3WriteHandle( IVSIS3LikeFSHandler* poFS,
588 const char* pszFilename,
589 IVSIS3LikeHandleHelper* poS3HandleHelper,
592 ~VSIS3WriteHandle()
override;
596 size_t Read(
void *pBuffer,
size_t nSize,
size_t nMemb )
override;
597 size_t Write(
const void *pBuffer,
size_t nSize,
size_t nMemb )
override;
599 int Close()
override;
601 bool IsOK() {
return m_bUseChunked || m_pabyBuffer !=
nullptr; }
614 VSICurlFilesystemHandlerBase* m_poFS =
nullptr;
619 int m_nBufferOff = 0;
620 int m_nBufferSize = 0;
621 int m_nBufferOffReadCallback = 0;
622 bool m_bClosed =
false;
623 GByte *m_pabyBuffer =
nullptr;
624 bool m_bError =
false;
626 static size_t ReadCallBackBuffer(
char *buffer,
size_t size,
627 size_t nitems,
void *instream );
628 virtual bool Send(
bool bIsLastBlock) = 0;
631 VSIAppendWriteHandle( VSICurlFilesystemHandlerBase* poFS,
632 const char* pszFSPrefix,
633 const char* pszFilename,
635 virtual ~VSIAppendWriteHandle();
639 size_t Read(
void *pBuffer,
size_t nSize,
size_t nMemb )
override;
640 size_t Write(
const void *pBuffer,
size_t nSize,
size_t nMemb )
override;
642 int Close()
override;
644 bool IsOK() {
return m_pabyBuffer !=
nullptr; }
651 struct CurlRequestHelper
653 WriteFuncStruct sWriteFuncData{};
654 WriteFuncStruct sWriteFuncHeaderData{};
655 char szCurlErrBuf[CURL_ERROR_SIZE+1] = {};
658 ~CurlRequestHelper();
659 long perform(CURL* hCurlHandle,
660 struct curl_slist* headers,
661 VSICurlFilesystemHandlerBase *poFS,
662 IVSIS3LikeHandleHelper *poS3HandleHelper);
669 class NetworkStatisticsLogger
671 static int gnEnabled;
672 static NetworkStatisticsLogger gInstance;
674 NetworkStatisticsLogger() =
default;
676 std::mutex m_mutex{};
685 GIntBig nGETDownloadedBytes = 0;
687 GIntBig nPOSTDownloadedBytes = 0;
688 GIntBig nPOSTUploadedBytes = 0;
691 enum class ContextPathType
698 struct ContextPathItem
700 ContextPathType eType;
703 ContextPathItem(ContextPathType eTypeIn,
const CPLString& osNameIn):
704 eType(eTypeIn), osName(osNameIn) {}
706 bool operator< (
const ContextPathItem& other )
const
708 if(
static_cast<int>(eType) <
static_cast<int>(other.eType) )
710 if(
static_cast<int>(eType) >
static_cast<int>(other.eType) )
712 return osName < other.osName;
719 std::map<ContextPathItem, Stats> children{};
727 std::map<GIntBig, std::vector<ContextPathItem>> m_mapThreadIdToContextPath{};
729 static void ReadEnabled();
731 std::vector<Counters*> GetCountersForContext();
735 static inline bool IsEnabled()
741 return gnEnabled == TRUE;
744 static void EnterFileSystem(
const char* pszName);
746 static void LeaveFileSystem();
748 static void EnterFile(
const char* pszName);
750 static void LeaveFile();
752 static void EnterAction(
const char* pszName);
754 static void LeaveAction();
756 static void LogHEAD();
758 static void LogGET(
size_t nDownloadedBytes);
760 static void LogPUT(
size_t nUploadedBytes);
762 static void LogPOST(
size_t nUploadedBytes,
763 size_t nDownloadedBytes);
765 static void LogDELETE();
769 static CPLString GetReportAsSerializedJSON();
772 struct NetworkStatisticsFileSystem
774 inline explicit NetworkStatisticsFileSystem(
const char* pszName) {
775 NetworkStatisticsLogger::EnterFileSystem(pszName);
778 inline ~NetworkStatisticsFileSystem()
780 NetworkStatisticsLogger::LeaveFileSystem();
784 struct NetworkStatisticsFile
786 inline explicit NetworkStatisticsFile(
const char* pszName) {
787 NetworkStatisticsLogger::EnterFile(pszName);
790 inline ~NetworkStatisticsFile()
792 NetworkStatisticsLogger::LeaveFile();
796 struct NetworkStatisticsAction
798 inline explicit NetworkStatisticsAction(
const char* pszName) {
799 NetworkStatisticsLogger::EnterAction(pszName);
802 inline ~NetworkStatisticsAction()
804 NetworkStatisticsLogger::LeaveAction();
809 int VSICURLGetDownloadChunkSize();
811 void VSICURLInitWriteFuncStruct( WriteFuncStruct *psStruct,
813 VSICurlReadCbkFunc pfnReadCbk,
814 void *pReadCbkUserData );
815 size_t VSICurlHandleWriteFunc(
void *buffer,
size_t count,
816 size_t nmemb,
void *req );
817 void MultiPerform(CURLM* hCurlMultiHandle,
818 CURL* hEasyHandle =
nullptr);
819 void VSICURLResetHeaderAndWriterFunctions(CURL* hCurlHandle);
821 int VSICurlParseUnixPermissions(
const char* pszPermissions);
The CPLJSONArray class holds JSON object from CPLJSONDocument.
Definition: cpl_json.h:54
String list class designed around our use of C "char**" string lists.
Definition: cpl_string.h:442
Convenient string class based on std::string.
Definition: cpl_string.h:333
Virtual file handle.
Definition: cpl_vsi_virtual.h:56
Interface for read and write JSON documents.
Core portability definitions for CPL.
#define EQUAL(a, b)
Alias for strcasecmp() == 0.
Definition: cpl_port.h:568
#define CPL_DISALLOW_COPY_ASSIGN(ClassName)
Helper to remove the copy and assignment constructors so that the compiler will not generate the defa...
Definition: cpl_port.h:955
char ** CSLConstList
Type of a constant null-terminated list of nul terminated strings.
Definition: cpl_port.h:1169
unsigned char GByte
Unsigned byte type.
Definition: cpl_port.h:215
long long GIntBig
Large signed integer type (generally 64-bit integer type).
Definition: cpl_port.h:244
Various convenience functions for working with strings and string lists.
#define VSI_L_OFFSET_MAX
Maximum value for a file offset.
Definition: cpl_vsi.h:142
struct VSIDIR VSIDIR
Opaque type for a directory iterator.
Definition: cpl_vsi.h:323
struct VSI_STAT64_T VSIStatBufL
Type for VSIStatL()
Definition: cpl_vsi.h:195
FILE VSILFILE
Opaque type for a FILE that implements the VSIVirtualHandle API.
Definition: cpl_vsi.h:156
GUIntBig vsi_l_offset
Type for a file offset.
Definition: cpl_vsi.h:140