import {
replaceProjectIdToken,
MissingProjectIdError,
} from '@google-cloud/projectify';
import * as ent from 'ent';
import * as extend from 'extend';
import {AuthClient, GoogleAuth, GoogleAuthOptions} from 'google-auth-library';
import {CredentialBody} from 'google-auth-library';
import * as r from 'teeny-request';
import * as retryRequest from 'retry-request';
import {Duplex, DuplexOptions, Readable, Transform, Writable} from 'stream';
import {teenyRequest} from 'teeny-request';
import {Interceptor} from './service-object';
import {DEFAULT_PROJECT_ID_TOKEN} from './service';
const duplexify: DuplexifyConstructor = require('duplexify');
const requestDefaults = {
timeout: 60000,
gzip: true,
forever: true,
pool: {
maxSockets: Infinity,
},
};
const AUTO_RETRY_DEFAULT = true;
const MAX_RETRY_DEFAULT = 3;
export type ResponseBody = any;
export interface DuplexifyOptions extends DuplexOptions {
autoDestroy?: boolean;
end?: boolean;
}
export interface Duplexify extends Duplex {
readonly destroyed: boolean;
setWritable(writable: Writable | false | null): void;
setReadable(readable: Readable | false | null): void;
}
export interface DuplexifyConstructor {
obj(
writable?: Writable | false | null,
readable?: Readable | false | null,
options?: DuplexifyOptions
): Duplexify;
new (
writable?: Writable | false | null,
readable?: Readable | false | null,
options?: DuplexifyOptions
): Duplexify;
(
writable?: Writable | false | null,
readable?: Readable | false | null,
options?: DuplexifyOptions
): Duplexify;
}
export interface ParsedHttpRespMessage {
resp: r.Response;
err?: ApiError;
}
export interface MakeAuthenticatedRequest {
(reqOpts: DecorateRequestOptions): Duplexify;
(
reqOpts: DecorateRequestOptions,
options?: MakeAuthenticatedRequestOptions
): void | Abortable;
(
reqOpts: DecorateRequestOptions,
callback?: BodyResponseCallback
): void | Abortable;
(
reqOpts: DecorateRequestOptions,
optionsOrCallback?: MakeAuthenticatedRequestOptions | BodyResponseCallback
): void | Abortable | Duplexify;
getCredentials: (
callback: (err?: Error | null, credentials?: CredentialBody) => void
) => void;
authClient: GoogleAuth<AuthClient>;
}
export interface Abortable {
abort(): void;
}
export type AbortableDuplex = Duplexify & Abortable;
export interface PackageJson {
name: string;
version: string;
}
export interface MakeAuthenticatedRequestFactoryConfig
extends Omit<GoogleAuthOptions, 'authClient'> {
autoRetry?: boolean;
customEndpoint?: boolean;
useAuthWithCustomEndpoint?: boolean;
email?: string;
maxRetries?: number;
stream?: Duplexify;
authClient?: AuthClient | GoogleAuth;
projectIdRequired?: boolean;
}
export interface MakeAuthenticatedRequestOptions {
onAuthenticated: OnAuthenticatedCallback;
}
export interface OnAuthenticatedCallback {
(err: Error | null, reqOpts?: DecorateRequestOptions): void;
}
export interface GoogleErrorBody {
code: number;
errors?: GoogleInnerError[];
response: r.Response;
message?: string;
}
export interface GoogleInnerError {
reason?: string;
message?: string;
}
export interface MakeWritableStreamOptions {
connection?: {};
metadata?: {contentType?: string};
request?: r.Options;
makeAuthenticatedRequest(
reqOpts: r.OptionsWithUri,
fnobj: {
onAuthenticated(
err: Error | null,
authenticatedReqOpts?: r.Options
): void;
}
): void;
}
export interface DecorateRequestOptions extends r.CoreOptions {
autoPaginate?: boolean;
autoPaginateVal?: boolean;
objectMode?: boolean;
maxRetries?: number;
uri: string;
interceptors_?: Interceptor[];
shouldReturnStream?: boolean;
projectId?: string;
}
export interface ParsedHttpResponseBody {
body: ResponseBody;
err?: Error;
}
export class ApiError extends Error {
code?: number;
errors?: GoogleInnerError[];
response?: r.Response;
constructor(errorMessage: string);
constructor(errorBody: GoogleErrorBody);
constructor(errorBodyOrMessage?: GoogleErrorBody | string) {
super();
if (typeof errorBodyOrMessage !== 'object') {
this.message = errorBodyOrMessage || '';
return;
}
const errorBody = errorBodyOrMessage;
this.code = errorBody.code;
this.errors = errorBody.errors;
this.response = errorBody.response;
try {
this.errors = JSON.parse(this.response.body).error.errors;
} catch (e) {
this.errors = errorBody.errors;
}
this.message = ApiError.createMultiErrorMessage(errorBody, this.errors);
Error.captureStackTrace(this);
}
static createMultiErrorMessage(
err: GoogleErrorBody,
errors?: GoogleInnerError[]
): string {
const messages: Set<string> = new Set();
if (err.message) {
messages.add(err.message);
}
if (errors && errors.length) {
errors.forEach(({message}) => messages.add(message!));
} else if (err.response && err.response.body) {
messages.add(ent.decode(err.response.body.toString()));
} else if (!err.message) {
messages.add('A failure occurred during this request.');
}
let messageArr: string[] = Array.from(messages);
if (messageArr.length > 1) {
messageArr = messageArr.map((message, i) => ` ${i + 1}. ${message}`);
messageArr.unshift(
'Multiple errors occurred during the request. Please see the `errors` array for complete details.\n'
);
messageArr.push('\n');
}
return messageArr.join('\n');
}
}
export class PartialFailureError extends Error {
errors?: GoogleInnerError[];
response?: r.Response;
constructor(b: GoogleErrorBody) {
super();
const errorObject = b;
this.errors = errorObject.errors;
this.name = 'PartialFailureError';
this.response = errorObject.response;
this.message = ApiError.createMultiErrorMessage(errorObject, this.errors);
}
}
export interface BodyResponseCallback {
(err: Error | ApiError | null, body?: ResponseBody, res?: r.Response): void;
}
export interface RetryOptions {
retryDelayMultiplier?: number;
totalTimeout?: number;
maxRetryDelay?: number;
autoRetry?: boolean;
maxRetries?: number;
retryableErrorFn?: (err: ApiError) => boolean;
}
export interface MakeRequestConfig {
autoRetry?: boolean;
maxRetries?: number;
retries?: number;
retryOptions?: RetryOptions;
stream?: Duplexify;
shouldRetryFn?: (response?: r.Response) => boolean;
}
export class Util {
ApiError = ApiError;
PartialFailureError = PartialFailureError;
noop() {}
handleResp(
err: Error | null,
resp?: r.Response | null,
body?: ResponseBody,
callback?: BodyResponseCallback
) {
callback = callback || util.noop;
const parsedResp = extend(
true,
{err: err || null},
resp && util.parseHttpRespMessage(resp),
body && util.parseHttpRespBody(body)
);
if (!parsedResp.err && resp && typeof parsedResp.body === 'object') {
parsedResp.resp.body = parsedResp.body;
}
if (parsedResp.err && resp) {
parsedResp.err.response = resp;
}
callback(parsedResp.err, parsedResp.body, parsedResp.resp);
}
parseHttpRespMessage(httpRespMessage: r.Response) {
const parsedHttpRespMessage = {
resp: httpRespMessage,
} as ParsedHttpRespMessage;
if (httpRespMessage.statusCode < 200 || httpRespMessage.statusCode > 299) {
parsedHttpRespMessage.err = new ApiError({
errors: new Array<GoogleInnerError>(),
code: httpRespMessage.statusCode,
message: httpRespMessage.statusMessage,
response: httpRespMessage,
});
}
return parsedHttpRespMessage;
}
parseHttpRespBody(body: ResponseBody) {
const parsedHttpRespBody: ParsedHttpResponseBody = {
body,
};
if (typeof body === 'string') {
try {
parsedHttpRespBody.body = JSON.parse(body);
} catch (err) {
parsedHttpRespBody.body = body;
}
}
if (parsedHttpRespBody.body && parsedHttpRespBody.body.error) {
parsedHttpRespBody.err = new ApiError(parsedHttpRespBody.body.error);
}
return parsedHttpRespBody;
}
makeWritableStream(
dup: Duplexify,
options: MakeWritableStreamOptions,
onComplete?: Function
) {
onComplete = onComplete || util.noop;
const writeStream = new ProgressStream();
writeStream.on('progress', evt => dup.emit('progress', evt));
dup.setWritable(writeStream);
const defaultReqOpts = {
method: 'POST',
qs: {
uploadType: 'multipart',
},
timeout: 0,
maxRetries: 0,
};
const metadata = options.metadata || {};
const reqOpts = extend(true, defaultReqOpts, options.request, {
multipart: [
{
'Content-Type': 'application/json',
body: JSON.stringify(metadata),
},
{
'Content-Type': metadata.contentType || 'application/octet-stream',
body: writeStream,
},
],
}) as r.OptionsWithUri;
options.makeAuthenticatedRequest(reqOpts, {
onAuthenticated(err, authenticatedReqOpts) {
if (err) {
dup.destroy(err);
return;
}
const request = teenyRequest.defaults(requestDefaults);
request(authenticatedReqOpts!, (err, resp, body) => {
util.handleResp(err, resp, body, (err, data) => {
if (err) {
dup.destroy(err);
return;
}
dup.emit('response', resp);
onComplete!(data);
});
});
},
});
}
shouldRetryRequest(err?: ApiError) {
if (err) {
if ([408, 429, 500, 502, 503, 504].indexOf(err.code!) !== -1) {
return true;
}
if (err.errors) {
for (const e of err.errors) {
const reason = e.reason;
if (reason === 'rateLimitExceeded') {
return true;
}
if (reason === 'userRateLimitExceeded') {
return true;
}
if (reason && reason.includes('EAI_AGAIN')) {
return true;
}
}
}
}
return false;
}
makeAuthenticatedRequestFactory(
config: MakeAuthenticatedRequestFactoryConfig
) {
const googleAutoAuthConfig = extend({}, config);
if (googleAutoAuthConfig.projectId === DEFAULT_PROJECT_ID_TOKEN) {
delete googleAutoAuthConfig.projectId;
}
let authClient: GoogleAuth<AuthClient>;
if (googleAutoAuthConfig.authClient instanceof GoogleAuth) {
authClient = googleAutoAuthConfig.authClient;
} else {
const config = {
...googleAutoAuthConfig,
authClient: googleAutoAuthConfig.authClient,
};
authClient = new GoogleAuth(config);
}
function makeAuthenticatedRequest(
reqOpts: DecorateRequestOptions
): Duplexify;
function makeAuthenticatedRequest(
reqOpts: DecorateRequestOptions,
options?: MakeAuthenticatedRequestOptions
): void | Abortable;
function makeAuthenticatedRequest(
reqOpts: DecorateRequestOptions,
callback?: BodyResponseCallback
): void | Abortable;
function makeAuthenticatedRequest(
reqOpts: DecorateRequestOptions,
optionsOrCallback?: MakeAuthenticatedRequestOptions | BodyResponseCallback
): void | Abortable | Duplexify {
let stream: Duplexify;
let projectId: string;
const reqConfig = extend({}, config);
let activeRequest_: void | Abortable | null;
if (!optionsOrCallback) {
stream = duplexify();
reqConfig.stream = stream;
}
const options =
typeof optionsOrCallback === 'object' ? optionsOrCallback : undefined;
const callback =
typeof optionsOrCallback === 'function' ? optionsOrCallback : undefined;
async function setProjectId() {
projectId = await authClient.getProjectId();
}
const onAuthenticated = async (
err: Error | null,
authenticatedReqOpts?: DecorateRequestOptions
) => {
const authLibraryError = err;
const autoAuthFailed =
err &&
err.message.indexOf('Could not load the default credentials') > -1;
if (autoAuthFailed) {
authenticatedReqOpts = reqOpts;
}
if (!err || autoAuthFailed) {
try {
authenticatedReqOpts = util.decorateRequest(
authenticatedReqOpts!,
projectId
);
err = null;
} catch (e) {
if (e instanceof MissingProjectIdError) {
try {
await setProjectId();
authenticatedReqOpts = util.decorateRequest(
authenticatedReqOpts!,
projectId
);
err = null;
} catch (e) {
err = err || (e as Error);
}
} else {
err = err || (e as Error);
}
}
}
if (err) {
if (stream) {
stream.destroy(err);
} else {
const fn =
options && options.onAuthenticated
? options.onAuthenticated
: callback;
(fn as Function)(err);
}
return;
}
if (options && options.onAuthenticated) {
options.onAuthenticated(null, authenticatedReqOpts);
} else {
activeRequest_ = util.makeRequest(
authenticatedReqOpts!,
reqConfig,
(apiResponseError, ...params) => {
if (
apiResponseError &&
(apiResponseError as ApiError).code === 401 &&
authLibraryError
) {
apiResponseError = authLibraryError;
}
callback!(apiResponseError, ...params);
}
);
}
};
const prepareRequest = async () => {
try {
const getProjectId = async () => {
if (
config.projectId &&
config.projectId !== DEFAULT_PROJECT_ID_TOKEN
) {
return config.projectId;
}
if (config.projectIdRequired === false) {
return DEFAULT_PROJECT_ID_TOKEN;
}
return setProjectId();
};
const authorizeRequest = async () => {
if (
reqConfig.customEndpoint &&
!reqConfig.useAuthWithCustomEndpoint
) {
return reqOpts;
} else {
return authClient.authorizeRequest(reqOpts);
}
};
const [_projectId, authorizedReqOpts] = await Promise.all([
getProjectId(),
authorizeRequest(),
]);
if (_projectId) {
projectId = _projectId;
}
return onAuthenticated(
null,
authorizedReqOpts as DecorateRequestOptions
);
} catch (e) {
return onAuthenticated(e as Error);
}
};
prepareRequest();
if (stream!) {
return stream!;
}
return {
abort() {
setImmediate(() => {
if (activeRequest_) {
activeRequest_.abort();
activeRequest_ = null;
}
});
},
};
}
const mar = makeAuthenticatedRequest as MakeAuthenticatedRequest;
mar.getCredentials = authClient.getCredentials.bind(authClient);
mar.authClient = authClient;
return mar;
}
makeRequest(
reqOpts: DecorateRequestOptions,
config: MakeRequestConfig,
callback: BodyResponseCallback
): void | Abortable {
let autoRetryValue = AUTO_RETRY_DEFAULT;
if (
config.autoRetry !== undefined &&
config.retryOptions?.autoRetry !== undefined
) {
throw new ApiError(
'autoRetry is deprecated. Use retryOptions.autoRetry instead.'
);
} else if (config.autoRetry !== undefined) {
autoRetryValue = config.autoRetry;
} else if (config.retryOptions?.autoRetry !== undefined) {
autoRetryValue = config.retryOptions.autoRetry;
}
let maxRetryValue = MAX_RETRY_DEFAULT;
if (config.maxRetries && config.retryOptions?.maxRetries) {
throw new ApiError(
'maxRetries is deprecated. Use retryOptions.maxRetries instead.'
);
} else if (config.maxRetries) {
maxRetryValue = config.maxRetries;
} else if (config.retryOptions?.maxRetries) {
maxRetryValue = config.retryOptions.maxRetries;
}
const options = {
request: teenyRequest.defaults(requestDefaults),
retries: autoRetryValue !== false ? maxRetryValue : 0,
noResponseRetries: autoRetryValue !== false ? maxRetryValue : 0,
shouldRetryFn(httpRespMessage: r.Response) {
const err = util.parseHttpRespMessage(httpRespMessage).err;
if (config.retryOptions?.retryableErrorFn) {
return err && config.retryOptions?.retryableErrorFn(err);
}
return err && util.shouldRetryRequest(err);
},
maxRetryDelay: config.retryOptions?.maxRetryDelay,
retryDelayMultiplier: config.retryOptions?.retryDelayMultiplier,
totalTimeout: config.retryOptions?.totalTimeout,
} as {} as retryRequest.Options;
if (typeof reqOpts.maxRetries === 'number') {
options.retries = reqOpts.maxRetries;
}
if (!config.stream) {
return retryRequest(reqOpts, options, (err, response, body) => {
util.handleResp(err, response as {} as r.Response, body, callback!);
});
}
const dup = config.stream as AbortableDuplex;
let requestStream: any;
const isGetRequest = (reqOpts.method || 'GET').toUpperCase() === 'GET';
if (isGetRequest) {
requestStream = retryRequest(reqOpts, options);
dup.setReadable(requestStream);
} else {
requestStream = options.request!(reqOpts);
dup.setWritable(requestStream);
}
requestStream
.on('error', dup.destroy.bind(dup))
.on('response', dup.emit.bind(dup, 'response'))
.on('complete', dup.emit.bind(dup, 'complete'));
dup.abort = requestStream.abort;
return dup;
}
decorateRequest(reqOpts: DecorateRequestOptions, projectId: string) {
delete reqOpts.autoPaginate;
delete reqOpts.autoPaginateVal;
delete reqOpts.objectMode;
if (reqOpts.qs !== null && typeof reqOpts.qs === 'object') {
delete reqOpts.qs.autoPaginate;
delete reqOpts.qs.autoPaginateVal;
reqOpts.qs = replaceProjectIdToken(reqOpts.qs, projectId);
}
if (Array.isArray(reqOpts.multipart)) {
reqOpts.multipart = (reqOpts.multipart as []).map(part => {
return replaceProjectIdToken(part, projectId);
});
}
if (reqOpts.json !== null && typeof reqOpts.json === 'object') {
delete reqOpts.json.autoPaginate;
delete reqOpts.json.autoPaginateVal;
reqOpts.json = replaceProjectIdToken(reqOpts.json, projectId);
}
reqOpts.uri = replaceProjectIdToken(reqOpts.uri, projectId);
return reqOpts;
}
isCustomType(unknown: any, module: string) {
function getConstructorName(obj: Function) {
return obj.constructor && obj.constructor.name.toLowerCase();
}
const moduleNameParts = module.split('/');
const parentModuleName =
moduleNameParts[0] && moduleNameParts[0].toLowerCase();
const subModuleName =
moduleNameParts[1] && moduleNameParts[1].toLowerCase();
if (subModuleName && getConstructorName(unknown) !== subModuleName) {
return false;
}
let walkingModule = unknown;
while (true) {
if (getConstructorName(walkingModule) === parentModuleName) {
return true;
}
walkingModule = walkingModule.parent;
if (!walkingModule) {
return false;
}
}
}
getUserAgentFromPackageJson(packageJson: PackageJson) {
const hyphenatedPackageName = packageJson.name
.replace('@google-cloud', 'gcloud-node')
.replace('/', '-');
return hyphenatedPackageName + '/' + packageJson.version;
}
maybeOptionsOrCallback<T = {}, C = (err?: Error) => void>(
optionsOrCallback?: T | C,
cb?: C
): [T, C] {
return typeof optionsOrCallback === 'function'
? [{} as T, optionsOrCallback as C]
: [optionsOrCallback as T, cb as C];
}
}
class ProgressStream extends Transform {
bytesRead = 0;
_transform(chunk: any, encoding: string, callback: Function) {
this.bytesRead += chunk.length;
this.emit('progress', {bytesWritten: this.bytesRead, contentLength: '*'});
this.push(chunk);
callback();
}
}
const util = new Util();
export {util};