import Review from '../models/Review.js' import cheerio from "cheerio"; import axios from 'axios'; const search = async (req, res, next) => { console.log("1") for (let i = 1; (req.reviews).length > 10; i++) { try { console.log("2") let reviews = [] let content = [] const url = "https://www.google.com/search?q=" + encodeURI(req.query.keyword) + "+site%3Atistory.com&page_no=" + i const response1 = await axios.get(url) // console.log(response1.data) const $1 = cheerio.load(response1.data); $1('.kCrYT').each(async function (i) { console.log("3") const title = $1(this).find('h3').text() const searchParams = new URLSearchParams($1(this).find('a').attr('href')); const link = searchParams.get("/url?q") const summary = $1(this).find('.s3v9rd').find('.s3v9rd').text() Review.find({ link: link }, function (err, review) { if (!review) { content.push(getReview(link)) if (title) { reviews[i] = { title: title, link: link } } else if (summary) { reviews[i - 1] = { ...reviews[i - 1], summary: summary } reviews = reviews.filter(e => e) } } }) }) let promiseReview = await Promise.all(content) promiseReview = promiseReview.filter(e => typeof (e) === 'string') reviews.forEach(async (review, i) => { console.log("4") review["content"] = promiseReview[i] const reviews = new Review(review).save() reviews.keyword.push(req.query.keyword) req.reviews.push(reviews) }) console.log("5") res.send(reviews) } catch (error) { console.log("6") console.log(error) res.send(error) } } } //***************네이버 크롤링 할 때 ********************* */ // try { // let reviews = [] // let content = [] // const url = "https://search.naver.com/search.naver?where=view&sm=tab_jum&query=" + encodeURI('한라산') // const response1 = await axios.get(url) // // console.log(response1.data) // const $1 = cheerio.load(response1.data); // $1('._list').find('.total_wrap').each(async function (i) { // reviews[i] = { name: $1(this).find('.total_tit').text(), summery: $1(this).find('.dsc_txt').text(), link: $1(this).find('.total_tit').attr('href') } // }) // res.send(reviews) // } catch (error) { // console.log(error) // res.send(error) // } // } const getReview = async (link) => { if (link) { let content = '없음' const res = await axios.get(link) const $2 = cheerio.load(res.data); if ($2('.tt_article_useless_p_margin').text()) { content = $2('.tt_article_useless_p_margin').text() } return content } } const find = (req, res, next) => { console.log("7", req.body) if (req.body.db) { next() } Review.find({ keyword:{ $elemMatch: res.query.keyword } }, function (err, reviews) { if (reviews) { if (reviews.length > 10) res.send(reviews, { db: true }) req.reviews = reviews } next() }) } export default { search, find }