minipass-fetch/lib/index.js

'use strict'
const { URL } = require('url')
const http = require('http')
const https = require('https')
const zlib = require('minizlib')
const { Minipass } = require('minipass')

const Body = require('./body.js')
const { writeToStream, getTotalBytes } = Body
const Response = require('./response.js')
const Headers = require('./headers.js')
const { createHeadersLenient } = Headers
const Request = require('./request.js')
const { getNodeRequestOptions } = Request
const FetchError = require('./fetch-error.js')
const AbortError = require('./abort-error.js')

// XXX this should really be split up and unit-ized for easier testing
// and better DRY implementation of data/http request aborting
const fetch = async (url, opts) => {
  if (/^data:/.test(url)) {
    const request = new Request(url, opts)
    // delay 1 promise tick so that the consumer can abort right away
    return Promise.resolve().then(() => new Promise((resolve, reject) => {
      let type, data
      try {
        const { pathname, search } = new URL(url)
        const split = pathname.split(',')
        if (split.length < 2) {
          throw new Error('invalid data: URI')
        }
        const mime = split.shift()
        const base64 = /;base64$/.test(mime)
        type = base64 ? mime.slice(0, -1 * ';base64'.length) : mime
        const rawData = decodeURIComponent(split.join(',') + search)
        data = base64 ? Buffer.from(rawData, 'base64') : Buffer.from(rawData)
      } catch (er) {
        return reject(new FetchError(`[${request.method}] ${
          request.url} invalid URL, ${er.message}`, 'system', er))
      }

      const { signal } = request
      if (signal && signal.aborted) {
        return reject(new AbortError('The user aborted a request.'))
      }

      const headers = { 'Content-Length': data.length }
      if (type) {
        headers['Content-Type'] = type
      }
      return resolve(new Response(data, { headers }))
    }))
  }

  return new Promise((resolve, reject) => {
    // build request object
    const request = new Request(url, opts)
    let options
    try {
      options = getNodeRequestOptions(request)
    } catch (er) {
      return reject(er)
    }

    const send = (options.protocol === 'https:' ? https : http).request
    const { signal } = request
    let response = null
    const abort = () => {
      const error = new AbortError('The user aborted a request.')
      reject(error)
      if (Minipass.isStream(request.body) &&
          typeof request.body.destroy === 'function') {
        request.body.destroy(error)
      }
      if (response && response.body) {
        response.body.emit('error', error)
      }
    }

    if (signal && signal.aborted) {
      return abort()
    }

    const abortAndFinalize = () => {
      abort()
      finalize()
    }

    const finalize = () => {
      req.abort()
      if (signal) {
        signal.removeEventListener('abort', abortAndFinalize)
      }
      clearTimeout(reqTimeout)
    }

    // send request
    const req = send(options)

    if (signal) {
      signal.addEventListener('abort', abortAndFinalize)
    }

    let reqTimeout = null
    if (request.timeout) {
      req.once('socket', socket => {
        reqTimeout = setTimeout(() => {
          reject(new FetchError(`network timeout at: ${
            request.url}`, 'request-timeout'))
          finalize()
        }, request.timeout)
      })
    }

    req.on('error', er => {
      // if a 'response' event is emitted before the 'error' event, then by the
      // time this handler is run it's too late to reject the Promise for the
      // response. instead, we forward the error event to the response stream
      // so that the error will surface to the user when they try to consume
      // the body. this is done as a side effect of aborting the request except
      // for in windows, where we must forward the event manually, otherwise
      // there is no longer a ref'd socket attached to the request and the
      // stream never ends so the event loop runs out of work and the process
      // exits without warning.
      // coverage skipped here due to the difficulty in testing
      // istanbul ignore next
      if (req.res) {
        req.res.emit('error', er)
      }
      reject(new FetchError(`request to ${request.url} failed, reason: ${
        er.message}`, 'system', er))
      finalize()
    })

    req.on('response', res => {
      clearTimeout(reqTimeout)

      const headers = createHeadersLenient(res.headers)

      // HTTP fetch step 5
      if (fetch.isRedirect(res.statusCode)) {
        // HTTP fetch step 5.2
        const location = headers.get('Location')

        // HTTP fetch step 5.3
        let locationURL = null
        try {
          locationURL = location === null ? null : new URL(location, request.url).toString()
        } catch {
          // error here can only be invalid URL in Location: header
          // do not throw when options.redirect == manual
          // let the user extract the errorneous redirect URL
          if (request.redirect !== 'manual') {
            /* eslint-disable-next-line max-len */
            reject(new FetchError(`uri requested responds with an invalid redirect URL: ${location}`, 'invalid-redirect'))
            finalize()
            return
          }
        }

        // HTTP fetch step 5.5
        if (request.redirect === 'error') {
          reject(new FetchError('uri requested responds with a redirect, ' +
            `redirect mode is set to error: ${request.url}`, 'no-redirect'))
          finalize()
          return
        } else if (request.redirect === 'manual') {
          // node-fetch-specific step: make manual redirect a bit easier to
          // use by setting the Location header value to the resolved URL.
          if (locationURL !== null) {
            // handle corrupted header
            try {
              headers.set('Location', locationURL)
            } catch (err) {
              /* istanbul ignore next: nodejs server prevent invalid
                 response headers, we can't test this through normal
                 request */
              reject(err)
            }
          }
        } else if (request.redirect === 'follow' && locationURL !== null) {
          // HTTP-redirect fetch step 5
          if (request.counter >= request.follow) {
            reject(new FetchError(`maximum redirect reached at: ${
              request.url}`, 'max-redirect'))
            finalize()
            return
          }

          // HTTP-redirect fetch step 9
          if (res.statusCode !== 303 &&
              request.body &&
              getTotalBytes(request) === null) {
            reject(new FetchError(
              'Cannot follow redirect with body being a readable stream',
              'unsupported-redirect'
            ))
            finalize()
            return
          }

          // Update host due to redirection
          request.headers.set('host', (new URL(locationURL)).host)

          // HTTP-redirect fetch step 6 (counter increment)
          // Create a new Request object.
          const requestOpts = {
            headers: new Headers(request.headers),
            follow: request.follow,
            counter: request.counter + 1,
            agent: request.agent,
            compress: request.compress,
            method: request.method,
            body: request.body,
            signal: request.signal,
            timeout: request.timeout,
          }

          // if the redirect is to a new hostname, strip the authorization and cookie headers
          const parsedOriginal = new URL(request.url)
          const parsedRedirect = new URL(locationURL)
          if (parsedOriginal.hostname !== parsedRedirect.hostname) {
            requestOpts.headers.delete('authorization')
            requestOpts.headers.delete('cookie')
          }

          // HTTP-redirect fetch step 11
          if (res.statusCode === 303 || (
            (res.statusCode === 301 || res.statusCode === 302) &&
              request.method === 'POST'
          )) {
            requestOpts.method = 'GET'
            requestOpts.body = undefined
            requestOpts.headers.delete('content-length')
          }

          // HTTP-redirect fetch step 15
          resolve(fetch(new Request(locationURL, requestOpts)))
          finalize()
          return
        }
      } // end if(isRedirect)

      // prepare response
      res.once('end', () =>
        signal && signal.removeEventListener('abort', abortAndFinalize))

      const body = new Minipass()
      // if an error occurs, either on the response stream itself, on one of the
      // decoder streams, or a response length timeout from the Body class, we
      // forward the error through to our internal body stream. If we see an
      // error event on that, we call finalize to abort the request and ensure
      // we don't leave a socket believing a request is in flight.
      // this is difficult to test, so lacks specific coverage.
      body.on('error', finalize)
      // exceedingly rare that the stream would have an error,
      // but just in case we proxy it to the stream in use.
      res.on('error', /* istanbul ignore next */ er => body.emit('error', er))
      res.on('data', (chunk) => body.write(chunk))
      res.on('end', () => body.end())

      const responseOptions = {
        url: request.url,
        status: res.statusCode,
        statusText: res.statusMessage,
        headers: headers,
        size: request.size,
        timeout: request.timeout,
        counter: request.counter,
        trailer: new Promise(resolveTrailer =>
          res.on('end', () => resolveTrailer(createHeadersLenient(res.trailers)))),
      }

      // HTTP-network fetch step 12.1.1.3
      const codings = headers.get('Content-Encoding')

      // HTTP-network fetch step 12.1.1.4: handle content codings

      // in following scenarios we ignore compression support
      // 1. compression support is disabled
      // 2. HEAD request
      // 3. no Content-Encoding header
      // 4. no content response (204)
      // 5. content not modified response (304)
      if (!request.compress ||
          request.method === 'HEAD' ||
          codings === null ||
          res.statusCode === 204 ||
          res.statusCode === 304) {
        response = new Response(body, responseOptions)
        resolve(response)
        return
      }

      // Be less strict when decoding compressed responses, since sometimes
      // servers send slightly invalid responses that are still accepted
      // by common browsers.
      // Always using Z_SYNC_FLUSH is what cURL does.
      const zlibOptions = {
        flush: zlib.constants.Z_SYNC_FLUSH,
        finishFlush: zlib.constants.Z_SYNC_FLUSH,
      }

      // for gzip
      if (codings === 'gzip' || codings === 'x-gzip') {
        const unzip = new zlib.Gunzip(zlibOptions)
        response = new Response(
          // exceedingly rare that the stream would have an error,
          // but just in case we proxy it to the stream in use.
          body.on('error', /* istanbul ignore next */ er => unzip.emit('error', er)).pipe(unzip),
          responseOptions
        )
        resolve(response)
        return
      }

      // for deflate
      if (codings === 'deflate' || codings === 'x-deflate') {
        // handle the infamous raw deflate response from old servers
        // a hack for old IIS and Apache servers
        const raw = res.pipe(new Minipass())
        raw.once('data', chunk => {
          // see http://stackoverflow.com/questions/37519828
          const decoder = (chunk[0] & 0x0F) === 0x08
            ? new zlib.Inflate()
            : new zlib.InflateRaw()
          // exceedingly rare that the stream would have an error,
          // but just in case we proxy it to the stream in use.
          body.on('error', /* istanbul ignore next */ er => decoder.emit('error', er)).pipe(decoder)
          response = new Response(decoder, responseOptions)
          resolve(response)
        })
        return
      }

      // for br
      if (codings === 'br') {
        // ignoring coverage so tests don't have to fake support (or lack of) for brotli
        // istanbul ignore next
        try {
          var decoder = new zlib.BrotliDecompress()
        } catch (err) {
          reject(err)
          finalize()
          return
        }
        // exceedingly rare that the stream would have an error,
        // but just in case we proxy it to the stream in use.
        body.on('error', /* istanbul ignore next */ er => decoder.emit('error', er)).pipe(decoder)
        response = new Response(decoder, responseOptions)
        resolve(response)
        return
      }

      // otherwise, use response as-is
      response = new Response(body, responseOptions)
      resolve(response)
    })

    writeToStream(req, request)
  })
}

module.exports = fetch

fetch.isRedirect = code =>
  code === 301 ||
  code === 302 ||
  code === 303 ||
  code === 307 ||
  code === 308

fetch.Headers = Headers
fetch.Request = Request
fetch.Response = Response
fetch.FetchError = FetchError
fetch.AbortError = AbortError