diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 1b5151ecd3..77e02bc8fc 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -7,6 +7,8 @@ This file contains a high-level description of this package's evolution. Release ## 4.10.0 - TBD +* Add extra failure handling to the daos inferencing. See [Github 3238](https://github.com/Unidata/netcdf-c/pull/3238) for more information. +* Regularize, cleanup, and refactor various AWS features, especially WRT regularizing AWS-related constants. See [Github 3229](https://github.com/Unidata/netcdf-c/pull/3229) for more information. * Regularize, cleanup, and refactor various AWS features, especially WRT regularizing AWS-related constants. See [Github 3229](https://github.com/Unidata/netcdf-c/issues/3229) for more information. * Add extra failure handling to the daos inferencing. See [Github 3238](https://github.com/Unidata/netcdf-c/issues/3238) for more information. * Regularize, cleanup, and refactor various AWS features, especially WRT regularizing AWS-related constants. See [Github 3229](https://github.com/Unidata/netcdf-c/issues/3229) for more information. diff --git a/include/Makefile.am b/include/Makefile.am index 29bb2dd902..1eef81afbb 100644 --- a/include/Makefile.am +++ b/include/Makefile.am @@ -26,7 +26,8 @@ nc4internal.h nctime.h nc3internal.h onstack.h ncrc.h ncauth.h \ ncoffsets.h nctestserver.h nc4dispatch.h nc3dispatch.h ncexternl.h \ ncpathmgr.h ncindex.h hdf4dispatch.h hdf5internal.h nc_provenance.h \ hdf5dispatch.h ncmodel.h isnan.h nccrc.h ncexhash.h ncxcache.h \ -ncjson.h ncxml.h ncs3sdk.h ncproplist.h ncplugins.h ncutil.h ncglobal.h +ncjson.h ncxml.h ncs3sdk.h ncproplist.h ncplugins.h ncutil.h ncglobal.h \ +ncaws.h if USE_DAP noinst_HEADERS += ncdap.h diff --git a/include/ncauth.h b/include/ncauth.h index 6022afa3a3..45335fa7cc 100644 --- a/include/ncauth.h +++ b/include/ncauth.h @@ -61,7 +61,7 @@ extern void NC_authfree(NCauth*); extern char* NC_combinehostport(NCURI*); extern int NC_parsecredentials(const char* userpwd, char** userp, char** pwdp); -extern int NC_authgets3creds(NCauth* auth, const char* profile, const char** accessidp, const char** secretkeyp); +extern int NC_authgets3creds(NCauth* auth, const char* profile, const char** accessidp, const char** secretkeyp, const char** session_tokenp); #if defined(__cplusplus) } diff --git a/include/ncaws.h b/include/ncaws.h new file mode 100644 index 0000000000..71d0ae1a0c --- /dev/null +++ b/include/ncaws.h @@ -0,0 +1,98 @@ +/* + * Copyright 2018, University Corporation for Atmospheric Research + * See netcdf/COPYRIGHT file for copying and redistribution conditions. + */ + +#ifndef NCAWS_H +#define NCAWS_H 1 + +#include "ncexternl.h" + +#define AWSHOST ".amazonaws.com" +#define GOOGLEHOST "storage.googleapis.com" + +/* Define the "global" default region to be used if no other region is specified */ +#define AWS_GLOBAL_DEFAULT_REGION "us-east-1" + +/* Provide macros for the keys for the possible sources of + AWS values: getenv(), .aws profiles, .ncrc keys, and URL fragment keys +*/ + +/* AWS getenv() keys */ +#define AWS_ENV_CONFIG_FILE "AWS_CONFIG_FILE" +#define AWS_ENV_PROFILE "AWS_PROFILE" +#define AWS_ENV_REGION "AWS_REGION" +#define AWS_ENV_DEFAULT_REGION "AWS_DEFAULT_REGION" +#define AWS_ENV_ACCESS_KEY_ID "AWS_ACCESS_KEY_ID" +#define AWS_ENV_SECRET_ACCESS_KEY "AWS_SECRET_ACCESS_KEY" +#define AWS_ENV_SESSION_TOKEN "AWS_SESSION_TOKEN" + +/* AWS .rc keys */ +#define AWS_RC_CONFIG_FILE "AWS.CONFIG_FILE" +#define AWS_RC_PROFILE "AWS.PROFILE" +#define AWS_RC_REGION "AWS.REGION" +#define AWS_RC_DEFAULT_REGION "AWS.DEFAULT_REGION" +#define AWS_RC_ACCESS_KEY_ID "AWS.ACCESS_KEY_ID" +#define AWS_RC_SECRET_ACCESS_KEY "AWS.SECRET_ACCESS_KEY" +#define AWS_RC_SESSION_TOKEN "AWS.SESSION_TOKEN" + +/* Known .aws profile keys (lowercase) */ +#define AWS_PROF_REGION "region" +#define AWS_PROF_ACCESS_KEY_ID "aws_access_key_id" +#define AWS_PROF_SECRET_ACCESS_KEY "aws_secret_access_key" +#define AWS_PROF_SESSION_TOKEN "aws_session_token" + +/* AWS URI fragment keys */ +#define AWS_FRAG_CONFIG_FILE AWS_RC_CONFIG_FILE +#define AWS_FRAG_PROFILE AWS_RC_PROFILE +#define AWS_FRAG_REGION AWS_RC_REGION +#define AWS_FRAG_DEFAULT_REGION AWS_RC_DEFAULT_REGION +#define AWS_FRAG_ACCESS_KEY_ID AWS_RC_ACCESS_KEY_ID +#define AWS_FRAG_SECRET_ACCESS_KEY AWS_RC_SECRET_ACCESS_KEY +#define AWS_FRAG_SESSION_TOKEN AWS_RC_SESSION_TOKEN + +typedef struct NCAWSPARAMS { /* AWS S3 specific parameters/defaults */ + char* config_file; + char* profile; + char* region; + char* default_region; + char* access_key_id; + char* secret_access_key; + char* session_token; +} NCAWSPARAMS; + +struct AWSentry { + char* key; + char* value; +}; + +struct AWSprofile { + char* name; + struct NClist* entries; /* NClist */ +}; + +/* Opaque */ +struct NCURI; +struct NCglobalstate; + +#ifdef __cplusplus +extern "C" { +#endif + +/* Extract AWS values from various sources */ +DECLSPEC void NC_awsglobal(void); +DECLSPEC void NC_awsnczfile(NCAWSPARAMS* zfileaws, struct NCURI* uri); +DECLSPEC void NC_awsenvironment(struct NCAWSPARAMS* aws); +DECLSPEC void NC_awsrc(struct NCAWSPARAMS* aws, struct NCURI* uri); +DECLSPEC void NC_awsfrag(struct NCAWSPARAMS* aws, struct NCURI* uri); + +DECLSPEC void NC_clearawsparams(struct NCAWSPARAMS*); +DECLSPEC NCAWSPARAMS NC_awsparams_empty(void); + +DECLSPEC int NC_aws_load_credentials(struct NCglobalstate* gstate); + +#ifdef __cplusplus +} +#endif + +#endif /*NCAWS_H*/ diff --git a/include/ncexternl.h b/include/ncexternl.h index 3fa65a7499..02c7e4ebdb 100644 --- a/include/ncexternl.h +++ b/include/ncexternl.h @@ -20,4 +20,16 @@ # define EXTERNL MSC_EXTRA extern #endif +#ifndef DECLSPEC +#ifdef DLL_NETCDF + #ifdef DLL_EXPORT /* define when building the library */ + #define DECLSPEC __declspec(dllexport) + #else + #define DECLSPEC __declspec(dllimport) + #endif +#else + #define DECLSPEC +#endif +#endif /*!DECLSPEC*/ + #endif /*NCEXTERNL_H*/ diff --git a/include/ncglobal.h b/include/ncglobal.h index 34fc82179d..f35549cc53 100644 --- a/include/ncglobal.h +++ b/include/ncglobal.h @@ -15,7 +15,6 @@ struct NClist; struct NCURI; struct NCRCinfo; struct NCZ_Plugin; -struct GlobalAWS; /**************************************************/ /* Begin to collect global state info in one place (more to do) */ @@ -39,13 +38,7 @@ typedef struct NCglobalstate { struct NCZ_Plugin** loaded_plugins; /*[H5Z_FILTER_MAX+1]*/ size_t loaded_plugins_max; /* plugin filter id index. 0 threshold and alignment explicitly set */ int threshold; @@ -58,9 +51,6 @@ typedef struct NCglobalstate { } chunkcache; } NCglobalstate; -/* Externally visible */ -typedef struct NCAWSPARAMS NCAWSPARAMS; - #if defined(__cplusplus) extern "C" { #endif @@ -69,8 +59,6 @@ extern "C" { struct NCglobalstate* NC_getglobalstate(void); void NC_freeglobalstate(void); -void NC_clearawsparams(struct GlobalAWS*); - #if defined(__cplusplus) } #endif diff --git a/include/ncs3sdk.h b/include/ncs3sdk.h index cdd1904d25..779f282576 100644 --- a/include/ncs3sdk.h +++ b/include/ncs3sdk.h @@ -46,7 +46,8 @@ #define AWS_FRAG_DEFAULT_REGION AWS_RC_DEFAULT_REGION /* Track the server type, if known */ -typedef enum NCS3SVC {NCS3UNK=0, /* unknown */ +typedef enum NCS3SVC { + NCS3UNK=0, /* unknown */ NCS3=1, /* s3.amazon.aws */ NCS3GS=2, /* storage.googleapis.com */ #ifdef NETCDF_ENABLE_ZOH @@ -56,6 +57,7 @@ typedef enum NCS3SVC {NCS3UNK=0, /* unknown */ /* Opaque Handles */ struct NClist; +struct AWSprofile; typedef struct NCS3INFO { char* host; /* non-null if other*/ @@ -66,16 +68,6 @@ typedef struct NCS3INFO { NCS3SVC svc; } NCS3INFO; -struct AWSentry { - char* key; - char* value; -}; - -struct AWSprofile { - char* name; - struct NClist* entries; /* NClist */ -}; - /* Opaque Types */ struct NClist; struct NCglobalstate; @@ -127,7 +119,6 @@ DECLSPEC void NC_s3getcredentials(const char *profile, const char **region, cons DECLSPEC int NC_authgets3profile(const char* profile, struct AWSprofile** profilep); DECLSPEC int NC_iss3(NCURI* uri, enum NCS3SVC*); DECLSPEC int NC_s3urlrebuild(NCURI* url, struct NCS3INFO* s3, NCURI** newurlp); -DECLSPEC int NC_aws_load_credentials(struct NCglobalstate* gstate); #ifdef __cplusplus } diff --git a/libdispatch/CMakeLists.txt b/libdispatch/CMakeLists.txt index 6d4d12b7fc..9fb0f0bfd8 100644 --- a/libdispatch/CMakeLists.txt +++ b/libdispatch/CMakeLists.txt @@ -15,6 +15,7 @@ target_sources(dispatch ncproplist.c ncindex.c dglobal.c + daws.c ) if (NETCDF_ENABLE_DLL) diff --git a/libdispatch/Makefile.am b/libdispatch/Makefile.am index 5bc40fe77e..c31f2e6908 100644 --- a/libdispatch/Makefile.am +++ b/libdispatch/Makefile.am @@ -22,7 +22,7 @@ ncbytes.c nchashmap.c nctime.c nc.c nclistmgr.c dauth.c doffsets.c \ dpathmgr.c dutil.c dreadonly.c dnotnc4.c dnotnc3.c dinfermodel.c \ daux.c dinstance.c dcrc32.c dcrc32.h dcrc64.c ncexhash.c ncxcache.c \ ncjson.c ds3util.c dparallel.c dmissing.c dinstance_intern.c \ -ncproplist.c ncindex.c dglobal.c +ncproplist.c ncindex.c dglobal.c daws.c # Add the utf8 codebase libdispatch_la_SOURCES += utf8proc.c utf8proc.h diff --git a/libdispatch/dauth.c b/libdispatch/dauth.c index 6e12ee3c72..e12e4708d1 100644 --- a/libdispatch/dauth.c +++ b/libdispatch/dauth.c @@ -20,6 +20,7 @@ See COPYRIGHT for license information. #include "nclog.h" #include "ncpathmgr.h" #include "ncs3sdk.h" +#include "ncaws.h" #include "ncauth.h" #ifdef _MSC_VER diff --git a/libdispatch/daws.c b/libdispatch/daws.c new file mode 100644 index 0000000000..f30899db0b --- /dev/null +++ b/libdispatch/daws.c @@ -0,0 +1,208 @@ +/* +Copyright (c) 1998-2018 University Corporation for Atmospheric Research/Unidata +See LICENSE.txt for license information. +*/ + +#include "config.h" +#include +#include +#include +#include + +#include "netcdf.h" +#include "ncglobal.h" +#include "ncaws.h" +#include "ncuri.h" +#include "ncrc.h" + +/**************************************************/ +/* Local Macros */ + +#ifndef REPLACE +#define REPLACE(dst,src) do{nullfree(dst); dst = nulldup(src);}while(0) +#endif + +/**************************************************/ +/* Forward */ + +static void NC_awsparamsmerge(struct NCAWSPARAMS* baseaws, struct NCAWSPARAMS* newaws); + +/**************************************************/ +/* Aws Param management */ + +void +NC_clearawsparams(struct NCAWSPARAMS* aws) +{ + assert(aws != NULL); + nullfree(aws->config_file); + nullfree(aws->profile); + nullfree(aws->region); + nullfree(aws->default_region); + nullfree(aws->access_key_id); + nullfree(aws->secret_access_key); + nullfree(aws->session_token); + memset(aws,0,sizeof(struct NCAWSPARAMS)); +} + +#if 0 +void +NC_cloneawsparams(struct NCAWSPARAMS* clone, struct NCAWSPARAMS* aws) +{ + NC_clearawsparams(clone); + clone->config_file = nulldup(aws->config_file); + clone->profile = nulldup(aws->profile); + clone->region = nulldup(aws->region); + clone->default_region = nulldup(aws->default_region); + clone->access_key_id = nulldup(aws->access_key_id); + clone->secret_access_key = nulldup(aws->secret_access_key); +} +#endif + +NCAWSPARAMS +NC_awsparams_empty(void) +{ + NCAWSPARAMS aws; + memset(&aws,0,sizeof(NCAWSPARAMS)); + return aws; +} + +/**************************************************/ +/* Capture environmental Info */ + +/* +When loading the globalstate AWS key values, load in the following order: +1. .rc file without URI patterns. +2. environment variables +Note: precedence order: 2 over 1. +*/ + +/* Load the globalstate.aws fields */ +void +NC_awsglobal(void) +{ + NCglobalstate* gs = NC_getglobalstate(); + NCAWSPARAMS aws; + + if(gs->aws == NULL) + gs->aws = (NCAWSPARAMS*)calloc(1,sizeof(NCAWSPARAMS)); + NC_clearawsparams(gs->aws); + + aws = NC_awsparams_empty(); + NC_clearawsparams(&aws); + + /* Get .rc information */ + NC_clearawsparams(&aws); + NC_awsrc(&aws,NULL); + NC_awsparamsmerge(gs->aws,&aws); + + /* Get environment information; overrides .rc file*/ + NC_clearawsparams(&aws); + NC_awsenvironment(&aws); + NC_awsparamsmerge(gs->aws,&aws); + NC_clearawsparams(&aws); + + /* Do some defaulting */ + if(gs->aws->default_region == NULL) gs->aws->default_region = nulldup(AWS_GLOBAL_DEFAULT_REGION); + if(gs->aws->region == NULL) gs->aws->region = nulldup(gs->aws->default_region); +} + +/* +When loading the NCZ_FILE_INFO_T AWS key values, load in the following order: +1. existing globalstate values +2. .rc file with URI patterns. +3. environment variables +4. URI fragment keys +Note: precedence order: 4 over 3 over 2 over 1. +*/ + +/* Load the NCZ_FILE_INFO_T aws fields*/ +void +NC_awsnczfile(NCAWSPARAMS* zfileaws, NCURI* uri) +{ + struct NCAWSPARAMS aws; + NCglobalstate* gs = NC_getglobalstate(); + + aws = NC_awsparams_empty(); + NC_clearawsparams(zfileaws); + + /* Initialize the aws from gs->aws */ + NC_awsparamsmerge(zfileaws,gs->aws); + + /* Get .rc information with uri from the open/create path */ + NC_clearawsparams(&aws); + NC_awsrc(&aws,uri); + NC_awsparamsmerge(zfileaws,&aws); + + /* Get environment information; overrides .rc file*/ + NC_clearawsparams(&aws); + NC_awsenvironment(&aws); + NC_awsparamsmerge(zfileaws,&aws); + + /* Get URI fragment information */ + NC_clearawsparams(&aws); + NC_awsfrag(&aws,uri); + NC_awsparamsmerge(zfileaws,&aws); + NC_clearawsparams(&aws); + + /* Do some defaulting */ + if(zfileaws->default_region == NULL) zfileaws->default_region = nulldup(AWS_GLOBAL_DEFAULT_REGION); + if(zfileaws->region == NULL) zfileaws->region = nulldup(zfileaws->default_region); +} + +static void +NC_awsparamsmerge(struct NCAWSPARAMS* baseaws, struct NCAWSPARAMS* newaws) +{ + assert(baseaws != NULL && newaws != NULL); + if(newaws->config_file != NULL) REPLACE(baseaws->config_file,newaws->config_file); + if(newaws->profile != NULL) REPLACE(baseaws->profile,newaws->profile); + if(newaws->region != NULL) REPLACE(baseaws->region,newaws->region); + if(newaws->default_region != NULL) REPLACE(baseaws->default_region,newaws->default_region); + if(newaws->access_key_id != NULL) REPLACE(baseaws->access_key_id,newaws->access_key_id); + if(newaws->secret_access_key != NULL) REPLACE(baseaws->secret_access_key,newaws->secret_access_key); + if(newaws->session_token != NULL) REPLACE(baseaws->session_token,newaws->session_token); +} + +/**************************************************/ +/* Lookup functors */ + +/* Collect aws params from env variables */ +void +NC_awsenvironment(struct NCAWSPARAMS* aws) +{ + NC_clearawsparams(aws); + aws->profile = nulldup(getenv(AWS_ENV_PROFILE)); + aws->config_file = nulldup(getenv(AWS_ENV_CONFIG_FILE)); + aws->region = nulldup(getenv(AWS_ENV_REGION)); + aws->default_region = nulldup(getenv(AWS_ENV_DEFAULT_REGION)); + aws->access_key_id = nulldup(getenv(AWS_ENV_ACCESS_KEY_ID)); + aws->secret_access_key = nulldup(getenv(AWS_ENV_SECRET_ACCESS_KEY)); + aws->session_token = nulldup(getenv(AWS_ENV_SESSION_TOKEN)); +} + +/* Setup aws params from .rc file */ +void +NC_awsrc(struct NCAWSPARAMS* aws, NCURI* uri) +{ + NC_clearawsparams(aws); + aws->profile = nulldup(NC_rclookupx(uri,AWS_RC_PROFILE)); + aws->config_file = nulldup(NC_rclookupx(uri,AWS_RC_CONFIG_FILE)); + aws->region = nulldup(NC_rclookupx(uri,AWS_RC_REGION)); + aws->default_region = nulldup(NC_rclookupx(uri,AWS_RC_DEFAULT_REGION)); + aws->access_key_id = nulldup(NC_rclookupx(uri,AWS_RC_ACCESS_KEY_ID)); + aws->secret_access_key = nulldup(NC_rclookupx(uri,AWS_RC_SECRET_ACCESS_KEY)); + aws->session_token = nulldup(NC_rclookupx(uri,AWS_RC_SESSION_TOKEN)); +} + +/* Setup aws params from URI fragment */ +void +NC_awsfrag(struct NCAWSPARAMS* aws, NCURI* uri) +{ + NC_clearawsparams(aws); + aws->profile = nulldup(ncurifragmentlookup(uri,AWS_FRAG_PROFILE)); + aws->config_file = nulldup(ncurifragmentlookup(uri,AWS_FRAG_CONFIG_FILE)); + aws->region = nulldup(ncurifragmentlookup(uri,AWS_FRAG_REGION)); + aws->default_region = nulldup(ncurifragmentlookup(uri,AWS_FRAG_DEFAULT_REGION)); + aws->access_key_id = nulldup(ncurifragmentlookup(uri,AWS_FRAG_ACCESS_KEY_ID)); + aws->secret_access_key = nulldup(ncurifragmentlookup(uri,AWS_FRAG_SECRET_ACCESS_KEY)); + aws->session_token = nulldup(ncurifragmentlookup(uri,AWS_FRAG_SESSION_TOKEN)); +} diff --git a/libdispatch/ddispatch.c b/libdispatch/ddispatch.c index 02aa53f9cb..9d1e4a3d78 100644 --- a/libdispatch/ddispatch.c +++ b/libdispatch/ddispatch.c @@ -13,6 +13,7 @@ See LICENSE.txt for license information. #include "ncpathmgr.h" #include "ncxml.h" #include "nc4internal.h" +#include "ncaws.h" /* Required for getcwd, other functions. */ #ifdef HAVE_UNISTD_H @@ -32,8 +33,6 @@ See LICENSE.txt for license information. #include "ncs3sdk.h" #endif -#define MAXPATH 1024 - /* Define vectors of zeros and ones for use with various nc_get_varX functions */ /* Note, this form of initialization fails under Cygwin */ size_t NC_coord_zero[NC_MAX_VAR_DIMS] = {0}; @@ -59,59 +58,10 @@ NCDISPATCH_initialize(void) NC_stride_one[i] = 1; } - globalstate = NC_getglobalstate(); /* will allocate and clear */ - - /* Capture temp dir*/ - { - char* tempdir = NULL; -#if defined _WIN32 || defined __MSYS__ || defined __CYGWIN__ - tempdir = getenv("TEMP"); -#else - tempdir = "/tmp"; -#endif - if(tempdir == NULL) { - fprintf(stderr,"Cannot find a temp dir; using ./\n"); - tempdir = "."; - } - globalstate->tempdir= strdup(tempdir); - } - - /* Capture $HOME */ - { -#if defined(_WIN32) && !defined(__MINGW32__) - char* home = getenv("USERPROFILE"); -#else - char* home = getenv("HOME"); -#endif - if(home == NULL) { - /* use cwd */ - home = malloc(MAXPATH+1); - NCgetcwd(home,MAXPATH); - } else - home = strdup(home); /* make it always free'able */ - assert(home != NULL); - NCpathcanonical(home,&globalstate->home); - nullfree(home); - } - - /* Capture $CWD */ - { - char cwdbuf[4096]; - - cwdbuf[0] = '\0'; - (void)NCgetcwd(cwdbuf,sizeof(cwdbuf)); - - if(strlen(cwdbuf) == 0) { - /* use tempdir */ - strcpy(cwdbuf, globalstate->tempdir); - } - globalstate->cwd = strdup(cwdbuf); - } - ncloginit(); - /* Now load RC Files */ - ncrc_initialize(); + /* will allocate, clear, and init some fields */ + (void)NC_getglobalstate(); /* Compute type alignments */ NC_compute_alignments(); diff --git a/libdispatch/dglobal.c b/libdispatch/dglobal.c index 721f4ed686..c15ef7e174 100644 --- a/libdispatch/dglobal.c +++ b/libdispatch/dglobal.c @@ -7,14 +7,22 @@ See LICENSE.txt for license information. #include #include #include +#include #include "netcdf.h" #include "ncglobal.h" +#include "ncaws.h" +#include "ncpathmgr.h" #include "nclist.h" #include "ncuri.h" #include "ncrc.h" +#include "nclog.h" #include "ncs3sdk.h" +/**************************************************/ + +#define MAXPATH 1024 + /**************************************************/ /* Global State constants and state */ @@ -23,6 +31,8 @@ static NCglobalstate* nc_globalstate = NULL; /* Forward */ static int NC_createglobalstate(void); +static void gs_chunkcache_init(NCglobalstate* gs); +static void gs_paths_init(NCglobalstate* gs); /** \defgroup global_state Global state functions. */ /** \{ @@ -51,21 +61,101 @@ NC_createglobalstate(void) memset(&nc_globalstate->chunkcache,0,sizeof(struct ChunkCache)); } - /* Get environment variables */ + /* Initialize chunk cache defaults */ + gs_chunkcache_init(nc_globalstate); + + /* Initialize various paths */ + gs_paths_init(nc_globalstate); + + /* Get .rc state */ if(getenv(NCRCENVIGNORE) != NULL) nc_globalstate->rcinfo->ignore = 1; tmp = getenv(NCRCENVRC); if(tmp != NULL && strlen(tmp) > 0) nc_globalstate->rcinfo->rcfile = strdup(tmp); + + ncrc_initialize(); + + /* Initialize aws defaults from .rc, env vars, aws profiles */ + NC_awsglobal(); + + /* load AWS Profiles: .aws/config &/ credentials */ + if(NC_aws_load_credentials(nc_globalstate)) { + nclog(NCLOGWARN,"AWS config file not loaded"); + } + +#if 0 /* Initialize chunk cache defaults */ nc_globalstate->chunkcache.size = DEFAULT_CHUNK_CACHE_SIZE; /**< Default chunk cache size. */ nc_globalstate->chunkcache.nelems = DEFAULT_CHUNKS_IN_CACHE; /**< Default chunk cache number of elements. */ nc_globalstate->chunkcache.preemption = DEFAULT_CHUNK_CACHE_PREEMPTION; /**< Default chunk cache preemption. */ - +#endif + done: return stat; } +/* Initialize chunk cache defaults */ +static void +gs_chunkcache_init(NCglobalstate* gs) +{ + gs->chunkcache.size = DEFAULT_CHUNK_CACHE_SIZE; /**< Default chunk cache size. */ + gs->chunkcache.nelems = DEFAULT_CHUNKS_IN_CACHE; /**< Default chunk cache number of elements. */ + gs->chunkcache.preemption = DEFAULT_CHUNK_CACHE_PREEMPTION;/**< Default chunk cache preemption. */ +} + +static void +gs_paths_init(NCglobalstate* gs) +{ + /* Capture temp dir*/ + { + char* tempdir = NULL; +#if defined _WIN32 || defined __MSYS__ || defined __CYGWIN__ + tempdir = getenv("TEMP"); +#else + tempdir = "/tmp"; +#endif + if(tempdir == NULL) { + fprintf(stderr,"Cannot find a temp dir; using ./\n"); + tempdir = "."; + } + gs->tempdir= strdup(tempdir); + } + + /* Capture $HOME */ + { +#if defined(_WIN32) && !defined(__MINGW32__) + char* home = getenv("USERPROFILE"); +#else + char* home = getenv("HOME"); +#endif + if(home == NULL) { + /* use cwd */ + home = malloc(MAXPATH+1); + NCgetcwd(home,MAXPATH); + } else + home = strdup(home); /* make it always free'able */ + assert(home != NULL); + NCpathcanonical(home,&gs->home); + nullfree(home); + } + + /* Capture $CWD */ + { + char cwdbuf[4096]; + + cwdbuf[0] = '\0'; + (void)NCgetcwd(cwdbuf,sizeof(cwdbuf)); + + if(strlen(cwdbuf) == 0) { + /* use tempdir */ + strcpy(cwdbuf, gs->tempdir); + } + gs->cwd = strdup(cwdbuf); + } + +} + /* Get global state */ NCglobalstate* NC_getglobalstate(void) @@ -84,7 +174,8 @@ NC_freeglobalstate(void) nullfree(gs->home); nullfree(gs->cwd); memset(&gs->chunkcache,0,sizeof(struct ChunkCache)); - NC_clearawsparams(&gs->aws); + NC_clearawsparams(gs->aws); + nullfree(gs->aws); if(gs->rcinfo) { NC_rcclear(gs->rcinfo); free(gs->rcinfo); @@ -96,15 +187,3 @@ NC_freeglobalstate(void) } /** \} */ - -void -NC_clearawsparams(struct GlobalAWS* aws) -{ - nullfree(aws->default_region); - nullfree(aws->config_file); - nullfree(aws->profile); - nullfree(aws->access_key_id); - nullfree(aws->secret_access_key); - memset(aws,0,sizeof(struct GlobalAWS)); -} - diff --git a/libdispatch/drc.c b/libdispatch/drc.c index 61df06bb83..185de0701d 100644 --- a/libdispatch/drc.c +++ b/libdispatch/drc.c @@ -28,8 +28,6 @@ See COPYRIGHT for license information. #include "ncdispatch.h" #include "ncutil.h" -#undef NOREAD - #undef DRCDEBUG #undef LEXDEBUG #undef PARSEDEBUG @@ -137,23 +135,15 @@ This is set by the environment variable NC_TEST_AWS_DIR. void ncrc_initialize(void) { - if(NCRCinitialized) return; - NCRCinitialized = 1; /* prevent recursion */ - -#ifndef NOREAD - { int stat = NC_NOERR; NCglobalstate* ncg = NC_getglobalstate(); + + if(NCRCinitialized) return; + NCRCinitialized = 1; /* prevent recursion */ /* Load entries */ if((stat = NC_rcload())) { nclog(NCLOGWARN,".rc loading failed"); } - /* Load .aws/config &/ credentials */ - if((stat = NC_aws_load_credentials(ncg))) { - nclog(NCLOGWARN,"AWS config file not loaded"); - } - } -#endif } static void @@ -293,10 +283,14 @@ char* NC_rclookupx(NCURI* uri, const char* key) { char* hostport = NULL; + char* path = NULL; char* result = NULL; - hostport = NC_combinehostport(uri); - result = NC_rclookup(key,hostport,uri->path); + if(uri != NULL) { + hostport = NC_combinehostport(uri); + path = uri->path; + } + result = NC_rclookup(key,hostport,path); nullfree(hostport); return result; } diff --git a/libdispatch/ds3util.c b/libdispatch/ds3util.c index 1de3cd2552..c916a52c1e 100644 --- a/libdispatch/ds3util.c +++ b/libdispatch/ds3util.c @@ -30,15 +30,27 @@ #include "nclog.h" #include "ncs3sdk.h" #include "ncutil.h" +#include "ncaws.h" #undef AWSDEBUG +/**************************************************/ +/* Local Macros */ + +#ifndef REPLACE +#define REPLACE(dst,src) do{nullfree(dst); dst = nulldup(src);}while(0) +#endif + /* Alternate .aws directory location */ #define NC_TEST_AWS_DIR "NC_TEST_AWS_DIR" +/**************************************************/ + enum URLFORMAT {UF_NONE=0, UF_VIRTUAL=1, UF_PATH=2, UF_S3=3, UF_OTHER=4}; -/* Read these files in order and later overriding earlier */ +/* Read these files in order and later overriding earlier; + trailing NULL is for any user-specified env variable, which takes precedence +*/ static const char* awsconfigfiles[] = {".aws/config",".aws/credentials",NULL}; #define NCONFIGFILES (sizeof(awsconfigfiles)/sizeof(char*)) @@ -61,15 +73,16 @@ NC_s3sdkenvironment(void) /* Get various environment variables as defined by the AWS sdk */ NCglobalstate* gs = NC_getglobalstate(); if(getenv(AWS_ENV_REGION)!=NULL) - gs->aws.default_region = nulldup(getenv(AWS_ENV_REGION)); + REPLACE(gs->aws->default_region,getenv(AWS_ENV_REGION)); else if(getenv(AWS_ENV_DEFAULT_REGION)!=NULL) - gs->aws.default_region = nulldup(getenv(AWS_ENV_DEFAULT_REGION)); - else if(gs->aws.default_region == NULL) - gs->aws.default_region = nulldup(AWS_GLOBAL_DEFAULT_REGION); - gs->aws.access_key_id = nulldup(getenv(AWS_ENV_ACCESS_KEY_ID)); - gs->aws.config_file = nulldup(getenv(AWS_ENV_CONFIG_FILE)); - gs->aws.profile = nulldup(getenv(AWS_ENV_PROFILE)); - gs->aws.secret_access_key = nulldup(getenv(AWS_ENV_SECRET_ACCESS_KEY)); + REPLACE(gs->aws->default_region,getenv(AWS_ENV_DEFAULT_REGION)); + else if(gs->aws->default_region == NULL) + REPLACE(gs->aws->default_region,AWS_GLOBAL_DEFAULT_REGION); + REPLACE(gs->aws->access_key_id,getenv(AWS_ENV_ACCESS_KEY_ID)); + REPLACE(gs->aws->config_file,getenv(AWS_ENV_CONFIG_FILE)); + REPLACE(gs->aws->profile,getenv(AWS_ENV_PROFILE)); + REPLACE(gs->aws->secret_access_key,getenv(AWS_ENV_SECRET_ACCESS_KEY)); + REPLACE(gs->aws->session_token,getenv(AWS_ENV_SESSION_TOKEN)); } /**************************************************/ @@ -513,12 +526,12 @@ int NC_aws_load_credentials(NCglobalstate* gstate) { int stat = NC_NOERR; + size_t i; NClist* profiles = nclistnew(); NCbytes* buf = ncbytesnew(); char path[8192]; const char* aws_root = getenv(NC_TEST_AWS_DIR); - const char* awscfg_local[NCONFIGFILES + 1]; /* +1 for the env variable */ - const char** awscfg = NULL; + const char* awscfg_local[NCONFIGFILES]; /* add a "no" credentials */ { @@ -528,15 +541,13 @@ NC_aws_load_credentials(NCglobalstate* gstate) nclistpush(profiles,noprof); noprof = NULL; } - awscfg = awsconfigfiles; - if((awscfg_local[0] = NC_getglobalstate()->aws.config_file)!=NULL) { - memcpy(&awscfg_local[1],awsconfigfiles,sizeof(char*)*NCONFIGFILES); - awscfg = awscfg_local; - } - for(;*awscfg;awscfg++) { + /* Make local copy list of places to look so we can add any place specified by env variable */ + memcpy(awscfg_local,awsconfigfiles,sizeof(char*)*NCONFIGFILES); + awscfg_local[2] = NC_getglobalstate()->aws->config_file; + for(i=0;i or Windows equivalent. */ - const char* cfg = *awscfg; - + const char* cfg = awscfg_local[i]; + if(cfg == NULL) continue; /* Ignore any NULL config files */ snprintf(path,sizeof(path),"%s%s%s", (aws_root?aws_root:gstate->home), (*cfg == '/'?"":"/"), @@ -559,7 +570,7 @@ NC_aws_load_credentials(NCglobalstate* gstate) struct AWSentry* entry = NULL; NCglobalstate* gs = NC_getglobalstate(); /* Verify that we can build a default */ - if(gs->aws.access_key_id != NULL && gs->aws.secret_access_key != NULL) { + if(gs->aws->access_key_id != NULL && gs->aws->secret_access_key != NULL) { /* Kill off any previous default profile */ for(i=nclistlength(profiles)-1;i>=0;i--) {/* walk backward because we are removing entries */ struct AWSprofile* prof = (struct AWSprofile*)nclistget(profiles,i); @@ -577,12 +588,17 @@ NC_aws_load_credentials(NCglobalstate* gstate) /* Create the entries for default */ if((entry = (struct AWSentry*)calloc(1,sizeof(struct AWSentry)))==NULL) {stat = NC_ENOMEM; goto done;} entry->key = strdup(AWS_PROF_ACCESS_KEY_ID); - entry->value = strdup(gs->aws.access_key_id); + entry->value = strdup(gs->aws->access_key_id); nclistpush(dfalt->entries,entry); entry = NULL; if((entry = (struct AWSentry*)calloc(1,sizeof(struct AWSentry)))==NULL) {stat = NC_ENOMEM; goto done;} entry->key = strdup(AWS_PROF_SECRET_ACCESS_KEY); - entry->value = strdup(gs->aws.secret_access_key); - nclistpush(dfalt->entries,entry); entry = NULL; + entry->value = strdup(gs->aws->secret_access_key); + if(gs->aws->session_token != NULL) { + if((entry = (struct AWSentry*)calloc(1,sizeof(struct AWSentry)))==NULL) {stat = NC_ENOMEM; goto done;} + entry->key = strdup(AWS_PROF_SESSION_TOKEN); + entry->value = strdup(gs->aws->session_token); + nclistpush(dfalt->entries,entry); entry = NULL; + } } } @@ -650,6 +666,8 @@ NC_s3profilelookup(const char* profile, const char* key, const char** valuep) if(valuep) *valuep = value; return stat; } + +#if 0 /** * Get the credentials for a given profile or load them from environment. @param profile name to use to look for credentials @@ -657,36 +675,38 @@ NC_s3profilelookup(const char* profile, const char* key, const char** valuep) @param accessid return accessid from progile or env @param accesskey return accesskey from profile or env */ -void NC_s3getcredentials(const char *profile, const char **region, const char** accessid, const char** accesskey) { +void +NC_s3getcredentials(const char *profile, const char **region, const char** accessid, const char** accesskey, const char** session_token) +{ if(profile != NULL && strcmp(profile,"no") != 0) { NC_s3profilelookup(profile, AWS_PROF_ACCESS_KEY_ID, accessid); NC_s3profilelookup(profile, AWS_PROF_SECRET_ACCESS_KEY, accesskey); NC_s3profilelookup(profile, AWS_PROF_REGION, region); + NC_s3profilelookup(profile, AWS_PROF_SESSION_TOKEN, session_token); } else { // We load from env if not in profile NCglobalstate* gstate = NC_getglobalstate(); - if(gstate->aws.access_key_id != NULL && accessid){ - *accessid = gstate->aws.access_key_id; + if(gstate->aws->access_key_id != NULL && accessid){ + *accessid = gstate->aws->access_key_id; } - if (gstate->aws.secret_access_key != NULL && accesskey){ - *accesskey = gstate->aws.secret_access_key; + if (gstate->aws->secret_access_key != NULL && accesskey){ + *accesskey = gstate->aws->secret_access_key; } - if(gstate->aws.default_region != NULL && region){ - *region = gstate->aws.default_region; + if(gstate->aws->default_region != NULL && region){ + *region = gstate->aws->default_region; } } } - +#endif /**************************************************/ /* Get the current active profile. The priority order is as follows: -1. aws.profile key in mode flags -2. aws.profile in .rc entries -3. AWS_PROFILE env variable -4. "default" -5. "no" -- meaning do not use any profile => no secret key +1. aws.profile key in URL fragment mode flags +2. aws->profile in NCglobalstate.aws +3. "default" +4. "no" -- meaning do not use any profile => no secret key @param uri uri with mode flags, may be NULL @param profilep return profile name here or NULL if none found @@ -708,9 +728,9 @@ NC_getactives3profile(NCURI* uri, const char** profilep) profile = NC_rclookupx(uri,AWS_RC_PROFILE); } - if(profile == NULL && gs->aws.profile != NULL) { - if((stat=NC_authgets3profile(gs->aws.profile,&ap))) goto done; - if(ap) profile = nulldup(gs->aws.profile); + if(profile == NULL && gs->aws->profile != NULL) { + if((stat=NC_authgets3profile(gs->aws->profile,&ap))) goto done; + if(ap) profile = nulldup(gs->aws->profile); } if(profile == NULL) { @@ -726,8 +746,9 @@ NC_getactives3profile(NCURI* uri, const char** profilep) #ifdef AWSDEBUG fprintf(stderr,">>> activeprofile = %s\n",(profile?profile:"null")); #endif - if(profilep) *profilep = profile; + if(profilep) {*profilep = profile; profile = NULL;} done: + nullfree(profile); return stat; } @@ -761,7 +782,7 @@ NC_getdefaults3region(NCURI* uri, const char** regionp) } } if(region == NULL) - region = (NC_getglobalstate()->aws.default_region ? NC_getglobalstate()->aws.default_region : "us-east-1"); /* Force use of the Amazon default */ + region = (NC_getglobalstate()->aws->default_region ? NC_getglobalstate()->aws->default_region : "us-east-1"); /* Force use of the Amazon default */ #ifdef AWSDEBUG fprintf(stderr,">>> activeregion = |%s|\n",region); #endif diff --git a/libdispatch/nch5s3comms.c b/libdispatch/nch5s3comms.c index 695c3c720f..a0d1503a8f 100644 --- a/libdispatch/nch5s3comms.c +++ b/libdispatch/nch5s3comms.c @@ -86,14 +86,12 @@ /* Necessary S3 headers */ #include -//#include -//#include -//#include #include "netcdf.h" #include "ncuri.h" #include "ncutil.h" #include "netcdf_vutils.h" +#include "ncaws.h" /*****************/ @@ -1056,10 +1054,16 @@ NCH5_s3comms_s3r_execute(s3r_t *handle, const char* url, * 2017-09-01 *---------------------------------------------------------------------------- */ + s3r_t * -NCH5_s3comms_s3r_open(const char* root, NCS3SVC svc, const char *region, const char *access_id, const char* access_key) +NCH5_s3comms_s3r_open(const char* root, NCS3SVC svc, NCAWSPARAMS* aws) + { int ret_value = SUCCEED; + const char *region = aws->region; + const char *access_id = aws->access_key_id; + const char* access_key = aws->secret_access_key; + const char* session_token = aws->session_token; size_t tmplen = 0; CURL *curlh = NULL; s3r_t *handle = NULL; @@ -1079,7 +1083,7 @@ NCH5_s3comms_s3r_open(const char* root, NCS3SVC svc, const char *region, const c handle = (s3r_t *)calloc(1,sizeof(s3r_t)); if (handle == NULL) - HGOTO_ERROR(H5E_ARGS, NC_ENOMEM, NULL, "could not malloc space for handle."); + HGOTO_ERROR(H5E_ARGS, NC_ENOMEM, NULL, "could not calloc space for handle."); handle->magic = S3COMMS_S3R_MAGIC; @@ -1105,27 +1109,15 @@ NCH5_s3comms_s3r_open(const char* root, NCS3SVC svc, const char *region, const c /* copy strings */ if(nulllen(region) != 0) { - tmplen = nulllen(region) + 1; - handle->region = (char *)malloc(sizeof(char) * tmplen); - if (handle->region == NULL) - HGOTO_ERROR(H5E_ARGS, NC_ENOMEM, NULL, "could not malloc space for handle region copy."); - memcpy(handle->region, region, tmplen); + handle->region = nulldup(region); } if(nulllen(access_id) != 0) { - tmplen = nulllen(access_id) + 1; - handle->accessid = (char *)malloc(sizeof(char) * tmplen); - if (handle->accessid == NULL) - HGOTO_ERROR(H5E_ARGS, NC_ENOMEM, NULL, "could not malloc space for handle ID copy."); - memcpy(handle->accessid, access_id, tmplen); + handle->accessid = nulldup(access_id); } if(nulllen(access_key) != 0) { - tmplen = nulllen(access_key) + 1; - handle->accesskey = (char *)malloc(sizeof(char) * tmplen); - if (handle->accesskey == NULL) - HGOTO_ERROR(H5E_ARGS, NC_ENOMEM, NULL, "could not malloc space for handle access key copy."); - memcpy(handle->accesskey, access_key, tmplen); + handle->accesskey = nulldup(access_key); } now = gmnow(); @@ -1145,8 +1137,18 @@ NCH5_s3comms_s3r_open(const char* root, NCS3SVC svc, const char *region, const c HGOTO_ERROR(H5E_ARGS, NC_EAUTH, NULL, "signing key cannot be null."); /* Compute the signing key */ - if (SUCCEED != NCH5_s3comms_signing_key(&signing_key, access_key, signingregion, iso8601now)) + if (SUCCEED != NCH5_s3comms_signing_key(&signing_key, access_key, region, session_token, iso8601now)) HGOTO_ERROR(H5E_ARGS, NC_EINVAL, NULL, "problem in NCH5_s3comms_s3comms_signing_key."); +#if S3COMMS_DEBUG_TRACE + { +int i; +fprintf(stderr,"@@@ signing_key: access_key=|%s| signingregion=|%s| iso8601now=|%s|\n", access_key, signingregion, iso8601now); +fprintf(stderr,"@@@\tsigning_key=|"); +for(i=0;i<(int)SHA256_DIGEST_LENGTH;i++) +fprintf(stderr,"%hhu",signing_key[i]); +fprintf(stderr,"|\n"); + } +#endif if (signing_key == NULL) HGOTO_ERROR(H5E_ARGS, NC_EAUTH, NULL, "signing key cannot be null."); handle->signing_key = signing_key; @@ -1159,6 +1161,7 @@ NCH5_s3comms_s3r_open(const char* root, NCS3SVC svc, const char *region, const c ************************/ curlh = curl_easy_init(); + if (curlh == NULL) HGOTO_ERROR(H5E_ARGS, NC_EINVAL, NULL, "problem creating curl easy handle!"); @@ -1168,6 +1171,11 @@ NCH5_s3comms_s3r_open(const char* root, NCS3SVC svc, const char *region, const c if (CURLE_OK != curl_easy_setopt(curlh, CURLOPT_FAILONERROR, 1L)) HGOTO_ERROR(H5E_ARGS, NC_EINVAL, NULL, "error while setting CURL option (CURLOPT_FAILONERROR)."); + if(getenv("CURLOPT_VERBOSE") != NULL) { + if (CURLE_OK != curl_easy_setopt(curlh, CURLOPT_VERBOSE, 1L)) + HGOTO_ERROR(H5E_ARGS, NC_EINVAL, NULL, "error while setting CURL option (CURLOPT_VERBOSE)."); + } + handle->curlhandle = curlh; /********************* @@ -1589,6 +1597,7 @@ NCH5_s3comms_HMAC_SHA256(const unsigned char *key, size_t key_len, const char *m * + aws_access_key_id * + aws_secret_access_key * + region + * + session_token (optional) * To be valid, the setting must begin the line with one of the keywords, * followed immediately by an equals sign '=', and have some data before * newline at end of line. @@ -1613,7 +1622,7 @@ NCH5_s3comms_HMAC_SHA256(const unsigned char *key, size_t key_len, const char *m */ static int H5FD__s3comms_load_aws_creds_from_file(FILE *file, const char *profile_name, char *key_id, char *access_key, - char *aws_region) + char *aws_region, char* session_token) { char profile_line[32]; char buffer[128]; @@ -1621,11 +1630,13 @@ H5FD__s3comms_load_aws_creds_from_file(FILE *file, const char *profile_name, cha AWS_PROF_REGION, AWS_PROF_ACCESS_KEY_ID, AWS_PROF_SECRET_ACCESS_KEY, + AWS_PROF_SESSION_TOKEN, }; char *const setting_pointers[] = { aws_region, key_id, access_key, + session_token, }; unsigned setting_count = 3; int ret_value = SUCCEED; @@ -1734,7 +1745,7 @@ H5FD__s3comms_load_aws_creds_from_file(FILE *file, const char *profile_name, cha */ int NCH5_s3comms_load_aws_profile(const char *profile_name, char *key_id_out, char *secret_access_key_out, - char *aws_region_out) + char *aws_region_out, char* aws_session_token_out) { int ret_value = SUCCEED; FILE *credfile = NULL; @@ -1760,7 +1771,7 @@ NCH5_s3comms_load_aws_profile(const char *profile_name, char *key_id_out, char * credfile = fopen(filepath, "r"); if (credfile != NULL) { if (H5FD__s3comms_load_aws_creds_from_file(credfile, profile_name, key_id_out, secret_access_key_out, - aws_region_out) != SUCCEED) + aws_region_out, aws_session_token_out) != SUCCEED) HGOTO_ERROR(H5E_ARGS, NC_EINVAL, FAIL, "unable to load from aws credentials"); if (fclose(credfile) == EOF) HGOTO_ERROR(H5E_FILE, NC_EACCESS, FAIL, "unable to close credentials file"); @@ -1775,14 +1786,15 @@ NCH5_s3comms_load_aws_profile(const char *profile_name, char *key_id_out, char * if (H5FD__s3comms_load_aws_creds_from_file( credfile, profile_name, (*key_id_out == 0) ? key_id_out : NULL, (*secret_access_key_out == 0) ? secret_access_key_out : NULL, - (*aws_region_out == 0) ? aws_region_out : NULL) != SUCCEED) + (*aws_region_out == 0) ? aws_region_out : NULL, + (*aws_session_token_out == 0) ? aws_session_token_out : NULL) != SUCCEED) HGOTO_ERROR(H5E_ARGS, NC_EINVAL, FAIL, "unable to load from aws config"); if (fclose(credfile) == EOF) HGOTO_ERROR(H5E_FILE, NC_EACCESS, FAIL, "unable to close config file"); credfile = NULL; } /* end if credential file opened */ - /* fail if not all three settings were loaded */ + /* fail if not all three settings were loaded; note session_token is optional */ if (*key_id_out == 0 || *secret_access_key_out == 0 || *aws_region_out == 0) ret_value = NC_EINVAL; @@ -1995,7 +2007,7 @@ NCH5_s3comms_percent_encode_char(char *repr, const unsigned char c, size_t *repr *---------------------------------------------------------------------------- */ int -NCH5_s3comms_signing_key(unsigned char **mdp, const char *secret, const char *region, const char *iso8601now) +NCH5_s3comms_signing_key(unsigned char **mdp, const char *secret, const char *region, const char* session_token, const char *iso8601now) { char *AWS4_secret = NULL; size_t AWS4_secret_len = 0; diff --git a/libdispatch/nch5s3comms.h b/libdispatch/nch5s3comms.h index 87b53cd988..f70620c122 100644 --- a/libdispatch/nch5s3comms.h +++ b/libdispatch/nch5s3comms.h @@ -75,6 +75,7 @@ struct CURL; struct NCURI; struct VString; +struct NCAWSPARAMS; /***************** * PUBLIC MACROS * @@ -440,6 +441,12 @@ typedef struct { * * Required to authenticate. * + * `session_token` (char *) + * + * Pointer to NULL-terminated string for short-term session token to access S3 resource. + * + * Optional ato authenticate. + * * `signing_key` (unsigned char *) * * Pointer to `SHA256_DIGEST_LENGTH`-long string for "reusable" signing @@ -460,6 +467,7 @@ typedef struct { char *region; char *accessid; char *accesskey; + char *session_token; char httpverb[S3COMMS_VERB_MAX]; unsigned char *signing_key; /*|signing_key| = SHA256_DIGEST_LENGTH*/ char iso8601now[ISO8601_SIZE]; @@ -502,7 +510,7 @@ EXTERNL hrb_t *NCH5_s3comms_hrb_init_request(const char *resource, const char *h * DECLARATION OF S3REQUEST ROUTINES * *************************************/ -EXTERNL s3r_t *NCH5_s3comms_s3r_open(const char* root, NCS3SVC svc, const char* region, const char* id, const char* access_key); +EXTERNL s3r_t *NCH5_s3comms_s3r_open(const char* root, NCS3SVC svc, struct NCAWSPARAMS* aws); EXTERNL int NCH5_s3comms_s3r_close(s3r_t *handle); @@ -538,13 +546,13 @@ EXTERNL int NCH5_s3comms_HMAC_SHA256(const unsigned char *key, size_t key_len, c size_t msg_len, char *dest); EXTERNL int NCH5_s3comms_load_aws_profile(const char *name, char *key_id_out, char *secret_access_key_out, - char *aws_region_out); + char *aws_region_out, char* aws_session_token_out); EXTERNL int NCH5_s3comms_nlowercase(char *dest, const char *s, size_t len); EXTERNL int NCH5_s3comms_percent_encode_char(char *repr, const unsigned char c, size_t *repr_len); -EXTERNL int NCH5_s3comms_signing_key(unsigned char **mdp, const char *secret, const char *region, +EXTERNL int NCH5_s3comms_signing_key(unsigned char **mdp, const char *secret, const char *region, const char* session_token, const char *iso8601now); EXTERNL int NCH5_s3comms_tostringtosign(struct VString* dest, const char *req_str, const char *now, diff --git a/libdispatch/ncs3sdk_h5.c b/libdispatch/ncs3sdk_h5.c index 35949c5e49..ac25379512 100644 --- a/libdispatch/ncs3sdk_h5.c +++ b/libdispatch/ncs3sdk_h5.c @@ -15,6 +15,7 @@ #include "ncrc.h" #include "ncxml.h" #include "ncutil.h" +#include "ncaws.h" #include "ncs3sdk.h" #include "nch5s3comms.h" @@ -117,7 +118,7 @@ static int queryinsert(NClist* list, char* ekey, char* evalue); static int ncs3_initialized = 0; static int ncs3_finalized = 0; -/*EXTERNL*/ int +int NC_s3sdkinitialize(void) { if(!ncs3_initialized) { @@ -131,7 +132,7 @@ NC_s3sdkinitialize(void) return NC_NOERR; } -/*EXTERNL*/ int +int NC_s3sdkfinalize(void) { if(!ncs3_finalized) { @@ -173,14 +174,16 @@ dumps3client(void* s3client0, const char* tag) /**************************************************/ -/*EXTERNL*/ void* +void* NC_s3sdkcreateclient(NCS3INFO* info) { int stat = NC_NOERR; const char* accessid = NULL; const char* accesskey = NULL; + const char* session_token = NULL; char* urlroot = NULL; NCS3CLIENT* s3client = NULL; + NCAWSPARAMS aws = NC_awsparams_empty(); NCTRACE(11,"info=%s",NC_s3dumps3info(info)); @@ -189,9 +192,14 @@ NC_s3sdkcreateclient(NCS3INFO* info) if(info->profile != NULL) { if((stat = NC_s3profilelookup(info->profile, AWS_PROF_ACCESS_KEY_ID, &accessid))) goto done; if((stat = NC_s3profilelookup(info->profile, AWS_PROF_SECRET_ACCESS_KEY, &accesskey))) goto done; + if((stat = NC_s3profilelookup(info->profile, AWS_PROF_SESSION_TOKEN, &session_token))) goto done; } if((s3client->rooturl = makes3rooturl(info))==NULL) {stat = NC_ENOMEM; goto done;} - s3client->h5s3client = NCH5_s3comms_s3r_open(s3client->rooturl,info->svc,info->region,accessid,accesskey); + aws.region = info->region; + aws.access_key_id = accessid; + aws.secret_access_key = accesskey; + aws.session_token = session_token; + s3client->h5s3client = NCH5_s3comms_s3r_open(s3client->rooturl,info->svc,&aws); if(s3client->h5s3client == NULL) {stat = NC_ES3; goto done;} done: @@ -204,7 +212,7 @@ NC_s3sdkcreateclient(NCS3INFO* info) return (void*)s3client; } -/*EXTERNL*/ int +int NC_s3sdkbucketexists(void* s3client0, const char* bucket, int* existsp, char** errmsgp) { int stat = NC_NOERR; @@ -225,7 +233,7 @@ NC_s3sdkbucketexists(void* s3client0, const char* bucket, int* existsp, char** e return NCUNTRACEX(stat,"exists=%d",PTRVAL(int,existsp,-1)); } -/*EXTERNL*/ int +int NC_s3sdkbucketcreate(void* s3client0, const char* region, const char* bucket, char** errmsgp) { int stat = NC_NOERR; @@ -239,7 +247,7 @@ NC_s3sdkbucketcreate(void* s3client0, const char* region, const char* bucket, ch return NCUNTRACE(stat); } -/*EXTERNL*/ int +int NC_s3sdkbucketdelete(void* s3client0, NCS3INFO* info, char** errmsgp) { int stat = NC_NOERR; @@ -263,7 +271,7 @@ NC_s3sdkbucketdelete(void* s3client0, NCS3INFO* info, char** errmsgp) @return NC_ENOOBJECT if object at key does not exist @return NC_EXXX return true error */ -/*EXTERNL*/ int +int NC_s3sdkinfo(void* s3client0, const char* bucket, const char* pathkey, size64_t* lenp, char** errmsgp) { int stat = NC_NOERR; @@ -289,7 +297,7 @@ NC_s3sdkinfo(void* s3client0, const char* bucket, const char* pathkey, size64_t* @return NC_NOERR if success @return NC_EXXX if fail */ -/*EXTERNL*/ int +int NC_s3sdkread(void* s3client0, const char* bucket, const char* pathkey, size64_t start, size64_t count, void* content, char** errmsgp) { int stat = NC_NOERR; @@ -316,7 +324,7 @@ NC_s3sdkread(void* s3client0, const char* bucket, const char* pathkey, size64_t For S3, I can see no way to do a byterange write; so we are effectively writing the whole object */ -/*EXTERNL*/ int +int NC_s3sdkwriteobject(void* s3client0, const char* bucket, const char* pathkey, size64_t count, const void* content, char** errmsgp) { int stat = NC_NOERR; @@ -340,7 +348,7 @@ NC_s3sdkwriteobject(void* s3client0, const char* bucket, const char* pathkey, s return NCUNTRACE(stat); } -/*EXTERNL*/ int +int NC_s3sdkclose(void* s3client0, char** errmsgp) { int stat = NC_NOERR; @@ -351,7 +359,7 @@ NC_s3sdkclose(void* s3client0, char** errmsgp) return NCUNTRACE(stat); } -/*EXTERNL*/ int +int NC_s3sdktruncate(void* s3client0, const char* bucket, const char* prefix, char** errmsgp) { int stat = NC_NOERR; @@ -462,7 +470,7 @@ Return a list of names of legal objects immediately below a specified key. In theory, the returned list should be sorted in lexical order, but it possible that it is not. */ -/*EXTERNL*/ int +int NC_s3sdklist(void* s3client0, const char* bucket, const char* prefixkey0, size_t* nkeysp, char*** keysp, char** errmsgp) { NCTRACE(11,"bucket=%s prefixkey0=%s",bucket,prefixkey0); @@ -474,14 +482,14 @@ Return a list of full keys of legal objects anywhere below a specified key. Not necessarily sorted. Essentially same as getkeys, but with no delimiter. */ -/*EXTERNL*/ int +int NC_s3sdklistall(void* s3client0, const char* bucket, const char* prefixkey0, size_t* nkeysp, char*** keysp, char** errmsgp) { NCTRACE(11,"bucket=%s prefixkey0=%s",bucket,prefixkey0); return NCUNTRACE(getkeys(s3client0, bucket, prefixkey0, NULL, nkeysp, keysp, errmsgp)); } -/*EXTERNL*/ int +int NC_s3sdkdeletekey(void* s3client0, const char* bucket, const char* pathkey, char** errmsgp) { int stat = NC_NOERR; diff --git a/libhdf5/hdf5open.c b/libhdf5/hdf5open.c index 7a44fbdb32..b4d4a22e67 100644 --- a/libhdf5/hdf5open.c +++ b/libhdf5/hdf5open.c @@ -16,6 +16,7 @@ #include "hdf5debug.h" #include "nc4internal.h" #include "ncrc.h" +#include "ncaws.h" #include "ncauth.h" #include "ncmodel.h" #include "ncpathmgr.h" @@ -889,6 +890,7 @@ nc4_open_file(const char *path, int mode, void* parameters, int ncid) H5FD_ros3_fapl_t fa; const char* awsaccessid0 = NULL; const char* awssecretkey0 = NULL; + const char* sessiontoken0 = NULL; const char* profile0 = NULL; int iss3 = NC_iss3(h5->uri,NULL); @@ -914,6 +916,8 @@ nc4_open_file(const char *path, int mode, void* parameters, int ncid) BAIL(retval); if((retval = NC_s3profilelookup(profile0,AWS_PROF_SECRET_ACCESS_KEY,&awssecretkey0))) BAIL(retval); + if((retval = NC_s3profilelookup(profile0,AWS_PROF_SESSION_TOKEN,&sessiontoken0))) + BAIL(retval); if(s3.region == NULL) s3.region = strdup(AWS_GLOBAL_DEFAULT_REGION); if(awsaccessid0 == NULL || awssecretkey0 == NULL ) { @@ -932,6 +936,11 @@ nc4_open_file(const char *path, int mode, void* parameters, int ncid) /* create and set fapl entry */ if(H5Pset_fapl_ros3(fapl_id, &fa) < 0) BAIL(NC_EHDFERR); + /* Set session token if defined */ + if(sessiontoken0 != NULL) { + if(H5Pset_fapl_ros3_token(fapl_id,sessiontoken0) < 0) + BAIL(NC_EHDFERR); + } } else #endif /*NETCDF_ENABLE_ROS3*/ {/* Configure FAPL to use our byte-range file driver */ diff --git a/nczarr_test/ut_test.c b/nczarr_test/ut_test.c index e4b75d96a4..8092eea15c 100644 --- a/nczarr_test/ut_test.c +++ b/nczarr_test/ut_test.c @@ -223,9 +223,9 @@ makeurl(const char* file, NCZM_IMPL impl, struct UTOptions* options) /* Assume that we have a complete url */ if(ncuriparse(file,&uri)) return NULL; if(options->profile) { - const char* profile = ncurifragmentlookup(uri,AWS_FRAG_PROFILE); + const char* profile = ncurifragmentlookup(uri,"aws.profile"); if(profile == NULL) { - ncurisetfragmentkey(uri,AWS_FRAG_PROFILE,options->profile); + ncurisetfragmentkey(uri,"aws.profile",options->profile); /* rebuild the url */ file = (const char*)ncuribuild(uri,NULL,NULL,NCURIALL); /* BAD but simple */ } diff --git a/unit_test/aws_config.c b/unit_test/aws_config.c index ad641cdba6..8b6b06c3ff 100644 --- a/unit_test/aws_config.c +++ b/unit_test/aws_config.c @@ -13,6 +13,7 @@ #include "ncs3sdk.h" #include "ncuri.h" #include "nc4internal.h" +#include "ncaws.h" NCS3INFO s3info; void* s3client = NULL; diff --git a/unit_test/test_s3sdk.c b/unit_test/test_s3sdk.c index 6475a2ba96..676f086f77 100644 --- a/unit_test/test_s3sdk.c +++ b/unit_test/test_s3sdk.c @@ -51,6 +51,7 @@ NCURI* purl = NULL; const char* activeprofile = NULL; const char* accessid = NULL; const char* accesskey = NULL; +const char* session_token = NULL; const char* newurl = NULL; NCS3INFO s3info; void* s3client = NULL; @@ -123,6 +124,7 @@ profilesetup(const char* url) CHECK(NC_getactives3profile(purl, &activeprofile)); CHECK(NC_s3profilelookup(activeprofile, "aws_access_key_id", &accessid)); CHECK(NC_s3profilelookup(activeprofile, "aws_secret_access_key", &accesskey)); + CHECK(NC_s3profilelookup(activeprofile, "aws_session_token", &session_token)); if(s3info.profile) free(s3info.profile); s3info.profile = (char*)nulldup(activeprofile); if(s3info.region == NULL) s3info.region = "";