Add authentication support for session tokens to AWS.

RELEASE_NOTES.md

@@ -7,6 +7,8 @@ This file contains a high-level description of this package's evolution. Release
 
 ## 4.10.0 - TBD
 
+* Add extra failure handling to the daos inferencing. See [Github 3238](https://github.com/Unidata/netcdf-c/pull/3238) for more information. 
+* Regularize, cleanup, and refactor various AWS features, especially WRT regularizing AWS-related constants. See [Github 3229](https://github.com/Unidata/netcdf-c/pull/3229) for more information. 
 * Regularize, cleanup, and refactor various AWS features, especially WRT regularizing AWS-related constants. See [Github 3229](https://github.com/Unidata/netcdf-c/issues/3229) for more information. 
 * Add extra failure handling to the daos inferencing. See [Github 3238](https://github.com/Unidata/netcdf-c/issues/3238) for more information. 
 * Regularize, cleanup, and refactor various AWS features, especially WRT regularizing AWS-related constants. See [Github 3229](https://github.com/Unidata/netcdf-c/issues/3229) for more information. 

include/Makefile.am

@@ -26,7 +26,8 @@ nc4internal.h nctime.h nc3internal.h onstack.h ncrc.h ncauth.h		\
 ncoffsets.h nctestserver.h nc4dispatch.h nc3dispatch.h ncexternl.h	\
 ncpathmgr.h ncindex.h hdf4dispatch.h hdf5internal.h nc_provenance.h	\
 hdf5dispatch.h ncmodel.h isnan.h nccrc.h ncexhash.h ncxcache.h          \
-ncjson.h ncxml.h ncs3sdk.h ncproplist.h ncplugins.h ncutil.h ncglobal.h
+ncjson.h ncxml.h ncs3sdk.h ncproplist.h ncplugins.h ncutil.h ncglobal.h \
+ncaws.h
 
 if USE_DAP
 noinst_HEADERS += ncdap.h

include/ncauth.h

@@ -61,7 +61,7 @@ extern void NC_authfree(NCauth*);
 extern char* NC_combinehostport(NCURI*);
 extern int NC_parsecredentials(const char* userpwd, char** userp, char** pwdp);
 
-extern int NC_authgets3creds(NCauth* auth, const char* profile, const char** accessidp, const char** secretkeyp);
+extern int NC_authgets3creds(NCauth* auth, const char* profile, const char** accessidp, const char** secretkeyp, const char** session_tokenp);
 
 #if defined(__cplusplus)
 }

include/ncaws.h

@@ -0,0 +1,98 @@
+/*
+ *	Copyright 2018, University Corporation for Atmospheric Research
+ *      See netcdf/COPYRIGHT file for copying and redistribution conditions.
+ */
+
+#ifndef NCAWS_H
+#define NCAWS_H 1
+
+#include "ncexternl.h"
+
+#define AWSHOST ".amazonaws.com"
+#define GOOGLEHOST "storage.googleapis.com"
+
+/* Define the "global" default region to be used if no other region is specified */
+#define AWS_GLOBAL_DEFAULT_REGION "us-east-1"
+
+/* Provide macros for the keys for the possible sources of
+   AWS values: getenv(), .aws profiles, .ncrc keys, and URL fragment keys
+*/
+
+/* AWS getenv() keys */
+#define AWS_ENV_CONFIG_FILE "AWS_CONFIG_FILE"
+#define AWS_ENV_PROFILE "AWS_PROFILE"
+#define AWS_ENV_REGION "AWS_REGION"
+#define AWS_ENV_DEFAULT_REGION "AWS_DEFAULT_REGION"
+#define AWS_ENV_ACCESS_KEY_ID "AWS_ACCESS_KEY_ID"
+#define AWS_ENV_SECRET_ACCESS_KEY "AWS_SECRET_ACCESS_KEY"
+#define AWS_ENV_SESSION_TOKEN "AWS_SESSION_TOKEN"
+
+/* AWS .rc keys */
+#define AWS_RC_CONFIG_FILE "AWS.CONFIG_FILE"
+#define AWS_RC_PROFILE "AWS.PROFILE"
+#define AWS_RC_REGION "AWS.REGION"
+#define AWS_RC_DEFAULT_REGION "AWS.DEFAULT_REGION"
+#define AWS_RC_ACCESS_KEY_ID "AWS.ACCESS_KEY_ID"
+#define AWS_RC_SECRET_ACCESS_KEY "AWS.SECRET_ACCESS_KEY"
+#define AWS_RC_SESSION_TOKEN "AWS.SESSION_TOKEN"
+
+/* Known .aws profile keys (lowercase) */
+#define AWS_PROF_REGION "region"
+#define AWS_PROF_ACCESS_KEY_ID "aws_access_key_id"
+#define AWS_PROF_SECRET_ACCESS_KEY "aws_secret_access_key"
+#define AWS_PROF_SESSION_TOKEN "aws_session_token"
+
+/* AWS URI fragment keys */
+#define AWS_FRAG_CONFIG_FILE AWS_RC_CONFIG_FILE
+#define AWS_FRAG_PROFILE AWS_RC_PROFILE
+#define AWS_FRAG_REGION AWS_RC_REGION
+#define AWS_FRAG_DEFAULT_REGION AWS_RC_DEFAULT_REGION
+#define AWS_FRAG_ACCESS_KEY_ID AWS_RC_ACCESS_KEY_ID
+#define AWS_FRAG_SECRET_ACCESS_KEY AWS_RC_SECRET_ACCESS_KEY
+#define AWS_FRAG_SESSION_TOKEN AWS_RC_SESSION_TOKEN
+
+typedef struct NCAWSPARAMS { /* AWS S3 specific parameters/defaults */
+    char* config_file;
+    char* profile;
+    char* region;
+    char* default_region;
+    char* access_key_id;
+    char* secret_access_key; 
+    char* session_token;
+} NCAWSPARAMS;
+
+struct AWSentry {
+    char* key;
+    char* value;
+};
+
+struct AWSprofile {
+    char* name;
+    struct NClist* entries; /* NClist<struct AWSentry*> */
+};
+
+/* Opaque */
+struct NCURI;
+struct NCglobalstate;
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* Extract AWS values from various sources */
+DECLSPEC void NC_awsglobal(void);
+DECLSPEC void NC_awsnczfile(NCAWSPARAMS* zfileaws, struct NCURI* uri);
+DECLSPEC void NC_awsenvironment(struct NCAWSPARAMS* aws);
+DECLSPEC void NC_awsrc(struct NCAWSPARAMS* aws, struct NCURI* uri);
+DECLSPEC void NC_awsfrag(struct NCAWSPARAMS* aws, struct NCURI* uri);
+
+DECLSPEC void NC_clearawsparams(struct NCAWSPARAMS*);
+DECLSPEC NCAWSPARAMS NC_awsparams_empty(void);
+
+DECLSPEC int NC_aws_load_credentials(struct NCglobalstate* gstate);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*NCAWS_H*/

include/ncexternl.h

@@ -20,4 +20,16 @@
 # define EXTERNL MSC_EXTRA extern
 #endif
 
+#ifndef DECLSPEC
+#ifdef DLL_NETCDF
+  #ifdef DLL_EXPORT /* define when building the library */
+    #define DECLSPEC __declspec(dllexport)
+  #else
+    #define DECLSPEC __declspec(dllimport)
+  #endif
+#else
+  #define DECLSPEC
+#endif
+#endif /*!DECLSPEC*/
+
 #endif /*NCEXTERNL_H*/

include/ncglobal.h

@@ -15,7 +15,6 @@ struct NClist;
 struct NCURI;
 struct NCRCinfo;
 struct NCZ_Plugin;
-struct GlobalAWS;
 
 /**************************************************/
 /* Begin to collect global state info in one place (more to do) */
@@ -39,13 +38,7 @@ typedef struct NCglobalstate {
 	struct NCZ_Plugin** loaded_plugins; /*[H5Z_FILTER_MAX+1]*/
 	size_t loaded_plugins_max; /* plugin filter id index. 0<loaded_plugins_max<=H5Z_FILTER_MAX */
     } zarr;
-    struct GlobalAWS { /* AWS S3 specific parameters/defaults */
-	char* default_region;
-	char* config_file;
-	char* profile;
-	char* access_key_id;
-	char* secret_access_key;
-    } aws;
+    struct NCAWSPARAMS* aws;
     struct Alignment { /* H5Pset_alignment parameters */
         int defined; /* 1 => threshold and alignment explicitly set */
 	int threshold;
@@ -58,9 +51,6 @@ typedef struct NCglobalstate {
     } chunkcache;
 } NCglobalstate;
 
-/* Externally visible */
-typedef struct NCAWSPARAMS NCAWSPARAMS;
-
 #if defined(__cplusplus)
 extern "C" {
 #endif
@@ -69,8 +59,6 @@ extern "C" {
 struct NCglobalstate* NC_getglobalstate(void);
 void NC_freeglobalstate(void);
 
-void NC_clearawsparams(struct GlobalAWS*);
-
 #if defined(__cplusplus)
 }
 #endif

include/ncs3sdk.h

@@ -46,7 +46,8 @@
 #define AWS_FRAG_DEFAULT_REGION AWS_RC_DEFAULT_REGION
 
 /* Track the server type, if known */
-typedef enum NCS3SVC {NCS3UNK=0, /* unknown */
+typedef enum NCS3SVC {
+	NCS3UNK=0, /* unknown */
 	NCS3=1,     /* s3.amazon.aws */
 	NCS3GS=2,   /* storage.googleapis.com */
 #ifdef NETCDF_ENABLE_ZOH
@@ -56,6 +57,7 @@ typedef enum NCS3SVC {NCS3UNK=0, /* unknown */
 
 /* Opaque Handles */
 struct NClist;
+struct AWSprofile;
 
 typedef struct NCS3INFO {
     char* host; /* non-null if other*/
@@ -66,16 +68,6 @@ typedef struct NCS3INFO {
     NCS3SVC svc;
 } NCS3INFO;
 
-struct AWSentry {
-    char* key;
-    char* value;
-};
-
-struct AWSprofile {
-    char* name;
-    struct NClist* entries; /* NClist<struct AWSentry*> */
-};
-
 /* Opaque Types */
 struct NClist;
 struct NCglobalstate;
@@ -127,7 +119,6 @@ DECLSPEC void NC_s3getcredentials(const char *profile, const char **region, cons
 DECLSPEC int NC_authgets3profile(const char* profile, struct AWSprofile** profilep);
 DECLSPEC int NC_iss3(NCURI* uri, enum NCS3SVC*);
 DECLSPEC int NC_s3urlrebuild(NCURI* url, struct NCS3INFO* s3, NCURI** newurlp);
-DECLSPEC int NC_aws_load_credentials(struct NCglobalstate* gstate);
 
 #ifdef __cplusplus
 }

libdispatch/CMakeLists.txt

@@ -15,6 +15,7 @@ target_sources(dispatch
     ncproplist.c 
     ncindex.c
     dglobal.c
+    daws.c
 )
 
 if (NETCDF_ENABLE_DLL)

libdispatch/Makefile.am

@@ -22,7 +22,7 @@ ncbytes.c nchashmap.c nctime.c nc.c nclistmgr.c dauth.c doffsets.c	\
 dpathmgr.c dutil.c dreadonly.c dnotnc4.c dnotnc3.c dinfermodel.c	\
 daux.c dinstance.c dcrc32.c dcrc32.h dcrc64.c ncexhash.c ncxcache.c	\
 ncjson.c ds3util.c dparallel.c dmissing.c dinstance_intern.c		\
-ncproplist.c ncindex.c dglobal.c
+ncproplist.c ncindex.c dglobal.c daws.c
 
 # Add the utf8 codebase
 libdispatch_la_SOURCES += utf8proc.c utf8proc.h

libdispatch/dauth.c

@@ -20,6 +20,7 @@ See COPYRIGHT for license information.
 #include "nclog.h"
 #include "ncpathmgr.h"
 #include "ncs3sdk.h"
+#include "ncaws.h"
 #include "ncauth.h"
 
 #ifdef _MSC_VER