Skip to content

Commit 2bc9c04

Browse files
snitin315snitin315
authored
chore: enable ssl verification for fetch-supporters (#7607)
1 parent 1237b6f commit 2bc9c04

File tree

1 file changed

+74
-96
lines changed

1 file changed

+74
-96
lines changed

‎src/utilities/fetch-supporters.mjs‎

Lines changed: 74 additions & 96 deletions
Original file line numberDiff line numberDiff line change
@@ -92,107 +92,85 @@ const nodeToSupporter = (node) => ({
9292
});
9393

9494
const getAllNodes = async (graphqlQuery, getNodes) => {
95-
// Store original value
96-
const originalTlsRejectUnauthorized =
97-
process.env.NODE_TLS_REJECT_UNAUTHORIZED;
98-
const isCI = process.env.CI === 'true' || (process.env.CI && process.env.VERCEL);
99-
100-
try {
101-
// Only disable SSL verification in local development
102-
if (!isCI) {
103-
process.env.NODE_TLS_REJECT_UNAUTHORIZED = '0';
104-
console.log('Running locally - SSL verification disabled');
95+
const body = {
96+
query: graphqlQuery,
97+
variables: {
98+
limit: graphqlPageSize,
99+
offset: 0,
100+
dateFrom: new Date(
101+
new Date().setFullYear(new Date().getFullYear() - 1)
102+
).toISOString(), // data from last year
103+
},
104+
};
105+
106+
let allNodes = [];
107+
108+
let limit = 10,
109+
remaining = 10,
110+
reset;
111+
if (process.env.OPENCOLLECTIVE_API_KEY) {
112+
limit = 100;
113+
remaining = 100;
114+
}
115+
// Handling pagination if necessary
116+
117+
while (true) {
118+
if (remaining === 0) {
119+
console.log(`Rate limit exceeded. Sleeping until ${new Date(reset)}.`);
120+
await new Promise((resolve) =>
121+
setTimeout(resolve, reset - Date.now() + 100)
122+
);
105123
}
106-
107-
const body = {
108-
query: graphqlQuery,
109-
variables: {
110-
limit: graphqlPageSize,
111-
offset: 0,
112-
dateFrom: new Date(
113-
new Date().setFullYear(new Date().getFullYear() - 1)
114-
).toISOString(), // data from last year
124+
const result = await fetch(graphqlEndpoint, {
125+
method: 'POST',
126+
body: JSON.stringify(body),
127+
headers: {
128+
'Content-Type': 'application/json',
115129
},
116-
};
117-
118-
let allNodes = [];
119-
120-
let limit = 10,
121-
remaining = 10,
122-
reset;
123-
if (process.env.OPENCOLLECTIVE_API_KEY) {
124-
limit = 100;
125-
remaining = 100;
126-
}
127-
// Handling pagination if necessary
128-
129-
while (true) {
130-
if (remaining === 0) {
131-
console.log(`Rate limit exceeded. Sleeping until ${new Date(reset)}.`);
132-
await new Promise((resolve) =>
133-
setTimeout(resolve, reset - Date.now() + 100)
134-
);
135-
}
136-
const fetchOptions = {
137-
method: 'POST',
138-
body: JSON.stringify(body),
139-
headers: {
140-
'Content-Type': 'application/json',
141-
},
142-
};
143-
144-
const result = await fetch(graphqlEndpoint, fetchOptions).then(
145-
async (response) => {
146-
if (response.headers.get('content-type').includes('json')) {
147-
const json = await response.json();
148-
console.log('json', json);
149-
if (json.error) {
150-
// when rate limit exceeded, api won't return headers data like x-ratelimit-limit, etc.
151-
remaining = 0;
152-
reset = Date.now() + 1000 * 60; // 1 minute
153-
} else {
154-
limit = response.headers.get('x-ratelimit-limit') * 1;
155-
remaining = response.headers.get('x-ratelimit-remaining') * 1;
156-
reset = response.headers.get('x-ratelimit-reset') * 1000;
157-
console.log(
158-
`Rate limit: ${remaining}/${limit} remaining. Reset in ${new Date(
159-
reset
160-
)}`
161-
);
162-
}
163-
return json;
164-
} else {
165-
// utilities/fetch-supporters: SyntaxError: Unexpected token < in JSON at position 0
166-
console.log('something wrong when fetching supporters');
167-
return {
168-
error: {
169-
message: await response.text(),
170-
},
171-
};
172-
}
173-
}
174-
);
175-
// when rate limit exceeded, api will return {error: {message: ''}}
176-
// but we could hopefully avoid rate limit by sleeping in the beginning of the loop
177-
// however, when there're multiple task running simultaneously, it's still possible to hit the rate limit
178-
if (result.error) {
179-
console.log('error', result.error);
180-
// let the loop continue
181-
} else {
182-
const nodes = getNodes(result.data);
183-
allNodes = [...allNodes, ...nodes];
184-
body.variables.offset += graphqlPageSize;
185-
if (nodes.length < graphqlPageSize) {
186-
return allNodes;
130+
}).then(async (response) => {
131+
if (response.headers.get('content-type').includes('json')) {
132+
const json = await response.json();
133+
console.log('json', json);
134+
if (json.error) {
135+
// when rate limit exceeded, api won't return headers data like x-ratelimit-limit, etc.
136+
remaining = 0;
137+
reset = Date.now() + 1000 * 60; // 1 minute
187138
} else {
188-
// more nodes to fetch
139+
limit = response.headers.get('x-ratelimit-limit') * 1;
140+
remaining = response.headers.get('x-ratelimit-remaining') * 1;
141+
reset = response.headers.get('x-ratelimit-reset') * 1000;
142+
console.log(
143+
`Rate limit: ${remaining}/${limit} remaining. Reset in ${new Date(
144+
reset
145+
)}`
146+
);
189147
}
148+
return json;
149+
} else {
150+
// utilities/fetch-supporters: SyntaxError: Unexpected token < in JSON at position 0
151+
console.log('something wrong when fetching supporters');
152+
return {
153+
error: {
154+
message: await response.text(),
155+
},
156+
};
157+
}
158+
});
159+
// when rate limit exceeded, api will return {error: {message: ''}}
160+
// but we could hopefully avoid rate limit by sleeping in the beginning of the loop
161+
// however, when there're multiple task running simultaneously, it's still possible to hit the rate limit
162+
if (result.error) {
163+
console.log('error', result.error);
164+
// let the loop continue
165+
} else {
166+
const nodes = getNodes(result.data);
167+
allNodes = [...allNodes, ...nodes];
168+
body.variables.offset += graphqlPageSize;
169+
if (nodes.length < graphqlPageSize) {
170+
return allNodes;
171+
} else {
172+
// more nodes to fetch
190173
}
191-
}
192-
} finally {
193-
// Only restore if we modified it
194-
if (!isCI) {
195-
process.env.NODE_TLS_REJECT_UNAUTHORIZED = originalTlsRejectUnauthorized;
196174
}
197175
}
198176
};

0 commit comments

Comments
 (0)