Scraping Google is a lost cause

This commit is contained in:
Dragon Fire
2020-01-16 12:15:44 -05:00
parent 353a80e77e
commit f2d4c42a7d
2 changed files with 2 additions and 27 deletions
-3
View File
@@ -1,3 +0,0 @@
{
"agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.118 Safari/537.36"
}
+2 -24
View File
@@ -1,8 +1,5 @@
const Command = require('../../structures/Command'); const Command = require('../../structures/Command');
const request = require('node-superfetch'); const request = require('node-superfetch');
const cheerio = require('cheerio');
const querystring = require('querystring');
const { agent } = require('../../assets/json/user-agent');
const { GOOGLE_KEY, CUSTOM_SEARCH_ID } = process.env; const { GOOGLE_KEY, CUSTOM_SEARCH_ID } = process.env;
module.exports = class GoogleCommand extends Command { module.exports = class GoogleCommand extends Command {
@@ -44,33 +41,14 @@ module.exports = class GoogleCommand extends Command {
let href; let href;
const nsfw = msg.channel.nsfw || false; const nsfw = msg.channel.nsfw || false;
try { try {
href = await this.searchGoogle(query, nsfw); href = await this.customSearch(query, nsfw);
} catch (err) { } catch (err) {
try { href = `http://lmgtfy.com/?iie=1&q=${encodeURIComponent(query)}`;
href = await this.customSearch(query, nsfw);
} catch (err2) {
href = `http://lmgtfy.com/?iie=1&q=${encodeURIComponent(query)}`;
}
} }
if (!href) return msg.say('Could not find any results.'); if (!href) return msg.say('Could not find any results.');
return msg.say(href); return msg.say(href);
} }
async searchGoogle(query, nsfw) {
const { text } = await request
.get('https://www.google.com/search')
.query({
safe: nsfw ? 'off' : 'on',
q: query
})
.set('User-Agent', agent);
const $ = cheerio.load(text);
let href = $('.r').first().find('a').first().attr('href');
if (!href) return null;
href = querystring.parse(href.replace('/url?', ''));
return href.q;
}
async customSearch(query, nsfw) { async customSearch(query, nsfw) {
const { body } = await request const { body } = await request
.get('https://www.googleapis.com/customsearch/v1') .get('https://www.googleapis.com/customsearch/v1')