Skip to content

Commit e774b2c

Browse files
laurirolauriro
committed
Use fetch for requests
1 parent 1916417 commit e774b2c

3 files changed

Lines changed: 94 additions & 28 deletions

File tree

‎s3.js‎

Lines changed: 47 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -13,14 +13,15 @@ module.exports = function S3(opts) {
1313
, hmac = (key, data) => crypto.createHmac("sha256", key).update(data).digest()
1414
, isFn = fn => typeof fn === "function"
1515
, isObj = obj => !!obj && obj.constructor === Object
16-
, isStream = stream => stream && isFn(stream.pipe)
16+
, isStream = stream => stream && isFn(stream.getReader || stream.getWriter || stream.pipe)
1717

1818
Object.assign(s3, {
1919
protocol: "https",
2020
region: "auto",
2121
endpoint: "s3." + opts.region + ".amazonaws.com",
2222
del: req.bind(s3, "DELETE", null),
2323
get: req.bind(s3, "GET", null),
24+
fetch: fakeFetch,
2425
list: function(path, opts, next) {
2526
if (isFn(opts)) {
2627
next = opts
@@ -109,33 +110,62 @@ module.exports = function S3(opts) {
109110
if (!isFn(next)) return new Promise(makeReq)
110111
makeReq(next.bind(null, null), next)
111112
function makeReq(resolve, reject) {
112-
var req = client.request(signed.url, { method: method, headers: headers }, handle)
113-
if (isStream(data)) data.pipe(req)
114-
else req.end(data)
115-
function handle(res) {
116-
res.on("error", reject)
117-
if (res.statusCode === 200 && isStream(next)) {
118-
res.pipe(next)
119-
return res.on("end", resolve)
113+
s3.fetch(signed.url, { method: method, headers: headers, body: data, duplex: "half" })
114+
.then(res => {
115+
if (res.ok && isStream(next)) {
116+
return res.body.pipeTo(next.pipe ? require("stream").Writable.toWeb(next) : next).then(resolve)
120117
}
121-
var data = ""
122-
res.on("data", chunk => data += chunk)
123-
res.on("end", () => {
124-
if (res.statusCode > 299) {
118+
res.text().then(data => {
119+
if (!res.ok) {
125120
data = method !== "HEAD" && parseXml(data).error || (
126121
path ? "The specified key does not exist." : "The specified bucket is not valid."
127122
)
128123
return reject(Error(data.message || data))
129124
}
130125
data = method === "HEAD" ? Object.assign({
131-
size: +res.headers["content-length"],
132-
mtime: new Date(res.headers["last-modified"]),
133-
}, res.headers) : parseXml(data)
126+
size: +res.headers.get("content-length"),
127+
mtime: new Date(res.headers.get("last-modified")),
128+
}, Object.fromEntries(res.headers.entries())) : parseXml(data)
129+
if (isStream(next)) next.end(data)
134130
resolve(data.listBucketResult || data.error || data)
135131
})
136-
}
132+
})
137133
}
138134
}
135+
function fakeFetch(url, opts) {
136+
return new Promise(function(resolve, reject) {
137+
var data = ""
138+
, addData = chunk => data += chunk
139+
, client = s3.client || require(url[4] === "s" ? "https" : "http")
140+
, req = client.request(url, opts, res => {
141+
res.on("error", reject).on("data", addData)
142+
resolve({
143+
ok: res.statusCode >= 200 && res.statusCode < 300,
144+
status: res.statusCode,
145+
text: () => new Promise(reso => {
146+
if (res.complete) reso(data)
147+
else res.on("end", () => reso(data))
148+
}),
149+
body: {
150+
pipeTo: stream => new Promise(reso => {
151+
if (!stream.pipe) stream = require("stream").Writable.fromWeb(stream)
152+
if (data) stream.write(data)
153+
res.off("data", addData).pipe(stream.on("close", reso))
154+
}),
155+
},
156+
headers: {
157+
entries: () => Object.entries(res.headers),
158+
get: name => res.headers[name]
159+
}
160+
})
161+
})
162+
if (isStream(opts.body)) {
163+
if (opts.body.pipe) opts.body.pipe(req)
164+
else require("stream").Readable.fromWeb(opts.body).pipe(req)
165+
}
166+
else req.end(opts.body)
167+
})
168+
}
139169
function parseXml(str) {
140170
var key, val
141171
, json = {}

‎test/index.js‎

Lines changed: 22 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ describe("S3 Mock", function() {
3333
},
3434
on: function(ev, fn) {
3535
listeners[ev] = fn
36+
return response
3637
}
3738
}
3839
return {
@@ -95,7 +96,8 @@ describe("S3 Mock", function() {
9596
"x-amz-date": "20220423T130929Z",
9697
"x-amz-content-sha256": "UNSIGNED-PAYLOAD",
9798
"Authorization": "AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20220423/eu-central-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=c0fc0f277bb492d0c91af0e9d27092591f1b13c8f20b058226fc91d7c013d1a5"
98-
}
99+
},
100+
body: null
99101
})
100102
assert.end()
101103
})
@@ -120,7 +122,8 @@ describe("S3 Mock", function() {
120122
"x-amz-date": "20220423T130929Z",
121123
"x-amz-content-sha256": "UNSIGNED-PAYLOAD",
122124
"Authorization": "AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20220423/eu-central-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=c0fc0f277bb492d0c91af0e9d27092591f1b13c8f20b058226fc91d7c013d1a5"
123-
}
125+
},
126+
body: null
124127
})
125128
assert.end()
126129
})
@@ -131,6 +134,7 @@ describe("S3 Mock", function() {
131134
var s3client = mockedClient(mock, { bucket: "buck-1", userAgent: "Dummy/1.0" })
132135
, streamReadable = { pipe: mock.fn(), read: mock.fn("Hello") }
133136
, stream = { pipe: mock.fn(), write: mock.fn() }
137+
stream.end = stream.write
134138

135139
s3client.put("file1.txt", streamReadable)
136140
s3client.get("file1.txt", stream)
@@ -167,7 +171,8 @@ describe("S3 Mock", function() {
167171
"x-amz-date": "20220423T130929Z",
168172
"x-amz-content-sha256": "UNSIGNED-PAYLOAD",
169173
"Authorization": "AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20220423/eu-central-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=0beac7539f091f6639599716f45739c55abf9544f39780bfae2bf92da2c35847"
170-
}
174+
},
175+
body: null
171176
})
172177
assert.end()
173178
})
@@ -191,7 +196,8 @@ describe("S3 Mock", function() {
191196
"x-amz-date": "20220423T130929Z",
192197
"x-amz-content-sha256": "UNSIGNED-PAYLOAD",
193198
"Authorization": "AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20220423/eu-central-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=f77ab4962fa59ed8823503a65a552cb51a5be67589504e314fdb21340a0d71a1"
194-
}
199+
},
200+
body: null
195201
})
196202
assert.end()
197203
})
@@ -215,7 +221,8 @@ describe("S3 Mock", function() {
215221
"x-amz-date": "20220423T130929Z",
216222
"x-amz-content-sha256": "UNSIGNED-PAYLOAD",
217223
"Authorization": "AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20220423/eu-central-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=a267714c9315bbd470114c8ffa9b378d15ef40d83af2e5ff8c294ff837044606"
218-
}
224+
},
225+
body: null
219226
})
220227
assert.end()
221228
})
@@ -239,7 +246,8 @@ describe("S3 Mock", function() {
239246
"x-amz-date": "20220423T130929Z",
240247
"x-amz-content-sha256": "64ec88ca00b268e5ba1a35678a1b5316d212f4f366b2477232534a8aeca37f3c",
241248
"Authorization": "AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20220423/eu-central-1/s3/aws4_request, SignedHeaders=content-length;host;x-amz-content-sha256;x-amz-date, Signature=7c3d078bb4da25b219264e060bea329836f27487477060d289daeca60435f9bc"
242-
}
249+
},
250+
body: data
243251
})
244252
assert.end()
245253
})
@@ -264,7 +272,8 @@ describe("S3 Mock", function() {
264272
"x-amz-content-sha256": "64ec88ca00b268e5ba1a35678a1b5316d212f4f366b2477232534a8aeca37f3c",
265273
"x-amz-meta-hello": "world",
266274
"Authorization": "AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20220423/eu-central-1/s3/aws4_request, SignedHeaders=content-length;host;x-amz-content-sha256;x-amz-date;x-amz-meta-hello, Signature=165648bd217c0e8a131fc06a65f95b85936fa15918ad7ee8e865848438e2c865"
267-
}
275+
},
276+
body: data
268277
})
269278
assert.end()
270279
})
@@ -313,7 +322,8 @@ describe("S3 Mock", function() {
313322
"x-amz-date": "20220423T130929Z",
314323
"x-amz-content-sha256": "UNSIGNED-PAYLOAD",
315324
"Authorization": "AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20220423/eu-central-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=94ff79fa807c4b865d476c359e54f58d279a6525fb1a0f1e0ecf0fd5f80bd61c"
316-
}
325+
},
326+
body: null
317327
})
318328
assert.end()
319329
})
@@ -388,7 +398,8 @@ describe("S3 Mock", function() {
388398
"x-amz-date": "20220423T130929Z",
389399
"x-amz-content-sha256": "UNSIGNED-PAYLOAD",
390400
"Authorization": "AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20220423/eu-central-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=e5c62c6e40d8ef5bfa00b425754f4f81e5399a28a0a69924507e55ac74bb87ba"
391-
}
401+
},
402+
body: null
392403
})
393404
assert.end()
394405
})
@@ -412,7 +423,8 @@ describe("S3 Mock", function() {
412423
"x-amz-date": "20220423T130929Z",
413424
"x-amz-content-sha256": "UNSIGNED-PAYLOAD",
414425
"Authorization": "AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20220423/eu-central-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=910e361836dd01e54e022fca928db767b4108554f7ed22bfaf901db40233bd25"
415-
}
426+
},
427+
body: null
416428
})
417429
assert.end()
418430
})

‎test/live.js‎

Lines changed: 25 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11

2+
describe("Native fetch: {0}", typeof fetch === "function" ? [true, false] : [false], nativeFetch=>{
23

34
describe("S3 live on {0} {1}", [
45
[ "AWS", "eu-north-1", true ],
@@ -37,6 +38,8 @@ describe("S3 live on {0} {1}", [
3738
, fileName = "gh-action-test1.txt"
3839
, content = "Hello " + Math.random()
3940

41+
if (nativeFetch) s3client.fetch = fetch
42+
4043
it("should upload a file", function(assert) {
4144
assert.setTimeout(5000)
4245
s3client.put(fileName, content, function(err, data) {
@@ -107,7 +110,7 @@ describe("S3 live on {0} {1}", [
107110
})
108111
})
109112

110-
it("should stream a file", async function(assert) {
113+
it("should stream a file to node stream", async function(assert) {
111114
assert.setTimeout(5000)
112115
var name = "./stream-" + fileName
113116
, writeTo = fs.createWriteStream(name)
@@ -125,6 +128,24 @@ describe("S3 live on {0} {1}", [
125128
fs.unlinkSync(name)
126129
})
127130

131+
it("should stream a file to web stream", async function(assert) {
132+
assert.setTimeout(5000)
133+
var name = "./stream2-" + fileName
134+
, writeTo = require("stream").Writable.toWeb(fs.createWriteStream(name))
135+
await s3client.get(fileName, writeTo)
136+
assert.equal(fs.readFileSync(name, "utf8"), content)
137+
138+
var readFrom = require("stream").Readable.toWeb(fs.createReadStream(name))
139+
, stat = fs.statSync(name)
140+
readFrom.length = stat.size
141+
142+
await s3client.put("streamed-" + fileName, readFrom)
143+
144+
assert.equal(await s3client.get("streamed-" + fileName), content)
145+
146+
fs.unlinkSync(name)
147+
})
148+
128149
describe("Presigned URL", function() {
129150
var content = "Hello " + Math.random()
130151
, fileName = "gh-action/signed-url.txt"
@@ -199,3 +220,6 @@ describe("S3 live on {0} {1}", [
199220
})
200221
})
201222

223+
224+
})
225+

0 commit comments

Comments
 (0)