Skip to content

Commit

Permalink
add user_agent query
Browse files Browse the repository at this point in the history
  • Loading branch information
@drawrowfly
drawrowfly committed Dec 16, 2020
1 parent d92c7ee commit 77fc0de
Show file tree
Hide file tree
Showing 3 changed files with 43 additions and 8 deletions.
14 changes: 12 additions & 2 deletions src/core/TikTok.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,17 @@ describe('TikTok Scraper MODULE(promise): user(valid input data)', () => {

it('getUserId should return a valid Object', async () => {
const userId: RequestQuery = await instance.getUserId();
expect(userId).toEqual({ count: 30, id: '107955', lang: '', maxCursor: 0, minCursor: 0, secUid: '', sourceType: 8, verifyFp: '' });
expect(userId).toEqual({
count: 30,
id: '107955',
lang: '',
maxCursor: 0,
minCursor: 0,
secUid: '',
sourceType: 8,
user_agent: 'Custom User-Agent',
verifyFp: '',
});
});

it('result should contain array value with the length 5', async () => {
Expand Down Expand Up @@ -277,7 +287,7 @@ describe('TikTok Scraper MODULE(promise): hashtag(valid input data)', () => {

it('getHashTagId should return a valid Object', async () => {
const hashtag: RequestQuery = await instance.getHashTagId();
expect(hashtag).toEqual({ aid: 1988, challengeID: '99770', count: 30, cursor: 0, verifyFp: '' });
expect(hashtag).toEqual({ aid: 1988, challengeID: '99770', count: 30, cursor: 0, user_agent: 'okhttp', verifyFp: '' });
});

// it('result should contain array value with the length 5', async () => {
Expand Down
36 changes: 30 additions & 6 deletions src/core/TikTok.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,8 @@ export class TikTokScraper extends EventEmitter {

private maxCursor: number;

private minCursor: number;

private noWaterMark: boolean;

private noDuplicates: string[];
Expand Down Expand Up @@ -174,6 +176,7 @@ export class TikTokScraper extends EventEmitter {
this.idStore = '';
this.noWaterMark = noWaterMark;
this.maxCursor = 0;
this.minCursor = 0;
this.noDuplicates = [];
this.timeout = timeout;
this.bulk = bulk;
Expand Down Expand Up @@ -451,7 +454,7 @@ export class TikTokScraper extends EventEmitter {
switch (this.scrapeType) {
case 'user':
this.getUserId()
.then(query => this.submitScrapingRequest({ ...query, maxCursor: this.maxCursor }))
.then(query => this.submitScrapingRequest({ ...query, maxCursor: this.maxCursor, minCursor: this.minCursor }))
.then(() => cb(null))
.catch(error => cb(error));
break;
Expand Down Expand Up @@ -494,7 +497,7 @@ export class TikTokScraper extends EventEmitter {
if (result.statusCode !== 0) {
throw new Error(`Can't scrape more posts`);
}
const { hasMore, maxCursor, cursor } = result;
const { hasMore, maxCursor, cursor, minCursor } = result;

if ((updatedApiResponse && !result.itemList) || (!updatedApiResponse && !result.items)) {
throw new Error('No more posts');
Expand All @@ -509,6 +512,7 @@ export class TikTokScraper extends EventEmitter {
throw new Error('Done');
}
this.maxCursor = parseInt(maxCursor === 'undefined' ? cursor : maxCursor, 10);
this.minCursor = minCursor ? parseInt(minCursor, 10) : 0;
} catch (error) {
throw error.message;
}
Expand Down Expand Up @@ -752,6 +756,7 @@ export class TikTokScraper extends EventEmitter {
},
json: true,
};

try {
const response = await this.request<T>(options);
return response;
Expand All @@ -774,6 +779,7 @@ export class TikTokScraper extends EventEmitter {
minCursor: 0,
maxCursor: 0,
verifyFp: this.verifyFp,
user_agent: this.headers['User-Agent'],
};
}

Expand All @@ -792,6 +798,7 @@ export class TikTokScraper extends EventEmitter {
count: 30,
cursor: 0,
verifyFp: '',
user_agent: this.headers['User-Agent'],
};
}

Expand All @@ -806,6 +813,7 @@ export class TikTokScraper extends EventEmitter {
cursor: 0,
aid: 1988,
verifyFp: this.verifyFp,
user_agent: this.headers['User-Agent'],
};
}
const id = encodeURIComponent(this.input);
Expand All @@ -826,6 +834,7 @@ export class TikTokScraper extends EventEmitter {
cursor: 0,
aid: 1988,
verifyFp: this.verifyFp,
user_agent: this.headers['User-Agent'],
};
} catch (error) {
throw error.message;
Expand All @@ -840,18 +849,22 @@ export class TikTokScraper extends EventEmitter {
return {
id: this.idStore ? this.idStore : this.input,
secUid: '',
aid: 1988,
sourceType: CONST.sourceType.user,
count: this.number > 30 ? 50 : 30,
count: 30,
minCursor: 0,
maxCursor: 0,
lang: '',
verifyFp: this.verifyFp,
user_agent: this.headers['User-Agent'],
};
}

const id = encodeURIComponent(this.input);
const query = {
uri: `${this.mainHost}node/share/user/@${id}?uniqueId=${id}&verifyFp=${this.verifyFp}`,
qs: {
user_agent: this.headers['User-Agent'],
},
method: 'GET',
json: true,
};
Expand All @@ -870,6 +883,7 @@ export class TikTokScraper extends EventEmitter {
maxCursor: 0,
lang: '',
verifyFp: this.verifyFp,
user_agent: this.headers['User-Agent'],
};
} catch (error) {
throw error.message;
Expand All @@ -884,13 +898,17 @@ export class TikTokScraper extends EventEmitter {
if (!this.input) {
throw `Username is missing`;
}
const query = {
const options = {
uri: `${this.mainHost}node/share/user/@${this.input}?uniqueId=${this.input}&verifyFp=${this.verifyFp}`,
qs: {
user_agent: this.headers['User-Agent'],
},
method: 'GET',
json: true,
};

try {
const response = await this.request<TikTokMetadata>(query);
const response = await this.request<TikTokMetadata>(options);

if (!response) {
throw new Error(`Can't find user: ${this.input}`);
Expand All @@ -914,6 +932,9 @@ export class TikTokScraper extends EventEmitter {
}
const query = {
uri: `${this.mainHost}node/share/tag/${this.input}?uniqueId=${this.input}`,
qs: {
user_agent: this.headers['User-Agent'],
},
method: 'GET',
json: true,
};
Expand Down Expand Up @@ -943,6 +964,9 @@ export class TikTokScraper extends EventEmitter {

const query = {
uri: `${this.mainHost}node/share/music/-${this.input}`,
qs: {
user_agent: this.headers['User-Agent'],
},
method: 'GET',
json: true,
};
Expand Down
1 change: 1 addition & 0 deletions src/types/TikTokApi.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,7 @@ export interface RequestQuery {
cursor?: number;
aid?: number;
appId?: number;
user_agent?: string;
}

export interface VideoProps {
Expand Down

0 comments on commit 77fc0de

Please sign in to comment.