Merge remote-tracking branch 'origin/cherry' into lsy

client/src/Components/Place.js

@@ -7,6 +7,7 @@ function Place(props) {
   const getReview = () => {
     axios.get(`/api/review/${props.search.name}`)
       .then(res => {
+        console.log("place res.data", res.data)
         setReviews(res.data)
       })
       .catch(err => {
@@ -38,6 +39,7 @@ function Place(props) {
               <Row className="mt-4">
                 <a href={review.link}>{review.title}</a>
                 <div>{review.summary}</div>
+                <div>{review.content}</div>
               </Row>
             )
           })}

server/controllers/review.controller copy.js

+import Review from '../models/Review.js'
+import cheerio, { html } from "cheerio";
+import jschardet from 'jschardet'
+import iconv from 'iconv'
+import fs from 'fs'
+import axios from 'axios';
+import { nextTick } from 'process';
+const Iconv = iconv.Iconv
+
+const search = async (req, res, next) => {
+    // const url = "https://www.google.com/search?q=" + encodeURI(req.params.search) + "+site%3Atistory.com&page_no=1"
+    const url = "https://www.google.com/search?q=" + encodeURI("한라산") + "+site%3Atistory.com&page_no=1"
+    let reviews = []
+    try {
+        axios.get(url)
+            // .then(anyToUtf8)
+            .then((html) => {
+                const $1 = cheerio.load(html.data);
+                $1('.kCrYT').each(function (i) {
+                    const searchParams = new URLSearchParams($1(this).find('a').attr('href'));
+                    const title = $1(this).find('h3').text()
+                    const link = searchParams.get("/url?q")
+                    const summary = $1(this).find('.s3v9rd').find('.s3v9rd').text()
+                    const response = axios.get(link)
+                    console.log(response)
+                    // const $2 = cheerio.load(toString(response.data));
+                    let content = "없음"
+                    // if ($2('.tt_article_useless_p_margin').text()) {
+                    //     content = $2('.tt_article_useless_p_margin').text()
+                    // }
+                    if (title) {
+                        reviews[i] = { title: title, link: link, content: content }
+                    } else if (summary) {
+                        reviews[i - 1] = { ...reviews[i - 1], summary: summary }
+                        reviews = reviews.filter(e => e)
+                    }
+
+                    // reviews.forEach((review, i) => {
+                    //     axios.get(review.link)
+                    //         .then((html) => {
+                    //             const $2 = cheerio.load(html.data);
+                    //             let content = '없음'
+                    //             if ($2('.tt_article_useless_p_margin').text()) {
+                    //                 content = $2('.tt_article_useless_p_margin').text()
+                    //             }
+                    //             // fs.writeFileSync(`tistory${i}.txt`, '\ufeff' + html.data, { encoding: 'utf8' });
+                    //             // console.log(`${i}번째, ${review.content}`)
+                    //             review["content"] = content
+                    //             // console.log(review)
+                    //         })
+                    //     })
+                    //     // const review = new Review(reviews).save()
+                })
+                res.send(reviews)
+            })
+    } catch (error) {
+        console.log(error)
+        res.status(500).send('리뷰 저장 에러')
+    }
+}
+
+const getReview = (link) => {
+    let content = '없음'
+    const res = axios.get(link)
+    console.log(res)
+    // const $2 = cheerio.load(res.data);
+    // if ($2('.tt_article_useless_p_margin').text()) {
+    //     content = $2('.tt_article_useless_p_margin').text()
+    // }
+
+    console.log(content)
+    return content
+}
+
+export default { search, getReview }
\ No newline at end of file

server/controllers/review.controller..js

+import Review from '../models/Review.js'
+import cheerio, { html } from "cheerio";
+import jschardet from 'jschardet'
+import iconv from 'iconv'
+import fs from 'fs'
+import axios from 'axios';
+import { nextTick } from 'process';
+import { response } from 'express';
+const Iconv = iconv.Iconv
+
+const search = async (req, res, next) => {
+    const url = "https://www.google.com/search?q=" + encodeURI(req.params.search) + "+site%3Atistory.com&page_no=1"
+    let reviews = []
+    try {
+        axios.get(url)
+            .then((response1) => {
+                const $1 = cheerio.load(response1.data);
+                $1('.kCrYT').each(function (i) {
+                    const searchParams = new URLSearchParams($1(this).find('a').attr('href'));
+                    const title = $1(this).find('h3').text()
+                    const link = searchParams.get("/url?q")
+                    const summary = $1(this).find('.s3v9rd').find('.s3v9rd').text()
+
+                    if (title) {
+                        reviews[i] = { title: title, link: link, }
+                    } else if (summary) {
+                        reviews[i - 1] = { ...reviews[i - 1], summary: summary }
+                        reviews = reviews.filter(e => e)
+                    }
+                    // const review = new Review(reviews).save()
+                })
+                res.send(reviews)
+            })
+    } catch (error) {
+
+        // reviews.forEach((review) => {
+        //     axios.get(review.link)
+        //         .then((response2) => {
+        //             const $2 = cheerio.load(response2.data);
+        //             let content = ''
+        //             if ($2('.tt_article_useless_p_margin').text()) {
+        //                 content = $2('.tt_article_useless_p_margin').text()
+        //             }
+        //             review = { content: content }
+        //             console.log("dkssud")
+        //         })
+        //         console.log("dkssud")
+        // })
+        // console.log(content)
+        res.status(500).send(error)
+    }
+}
+const getReview = async (link) => {
+    let content = '없음'
+    const res = await axios.get(link)
+    const $2 = cheerio.load(res.data);
+    if ($2('.tt_article_useless_p_margin').text()) {
+        content = $2('.tt_article_useless_p_margin').text()
+    }
+    console.log(content, "getReiview")
+    return content
+}
+
+export default { search, getReview }
\ No newline at end of file

server/controllers/review.controller.js

@@ -6,53 +6,95 @@ import cheerio from "cheerio";
 import axios from 'axios';
 // const Iconv = iconv.Iconv
 
-const search = async (req, res) => {
-    const url = "https://www.google.com/search?q=" + encodeURI(req.params.search) + "+site%3Atistory.com&page_no=1"
-    const editUrl = /(http(s)?:\/\/)([a-z0-9\w]+\.*)+[a-z0-9]{2,4}/gi
-    axios.get(url)
-        // .then(anyToUtf8)
-        .then((html) => {
-            // fs.writeFileSync("googlez.txt", '\ufeff' + html, { encoding: 'utf8' });
-            let $1 = cheerio.load(html.data);
+// const search = async (req, res) => {
+//     const url = "https://www.google.com/search?q=" + encodeURI(req.params.search) + "+site%3Atistory.com&page_no=1"
+//     const editUrl = /(http(s)?:\/\/)([a-z0-9\w]+\.*)+[a-z0-9]{2,4}/gi
+//     axios.get(url)
+//         // .then(anyToUtf8)
+//         .then((html) => {
+//             // fs.writeFileSync("googlez.txt", '\ufeff' + html, { encoding: 'utf8' });
+//             let $1 = cheerio.load(html.data);
+//             let reviews = []
+//             $1('.kCrYT').each(function (i) {
+//                 let link = ""
+//                 if ($1(this).find('a').attr('href')) {
+//                     link = ($1(this).find('a').attr('href')).match(editUrl)
+//                 }
+//                 reviews[i] = {
+//                     title: $1(this).find('h3').text(),
+//                     link: link,
+//                     summary: $1(this).find('.s3v9rd').text(),
+//                 }
+//             })
+//             // reviews.forEach((review, i) => {
+//             //     axios.get(review.link)
+//             //         .then((html) => {
+//             //             let $2 = cheerio.load(html.data);
+//             //             $2('').each(function(i){
+//             //                 review.content($2(this).find(''.text()))
+//             //             })
+//             //         })
+//             // })
+
+//             console.log(reviews)
+//             res.send(reviews)
+//         })
+//     // function anyToUtf8(str) {
+//     //     const { encoding } = jschardet.detect(str); // 웹페이지 문서의 인코딩 타입을 확인
+//     //     const iconv = new Iconv(encoding, "utf-8//translit//ignore"); // euc-kr 인코딩변환
+//     //     return iconv.convert(str).toString();
+//     // }
+//     // try {
+//     //     const newPlace = await new Place({
+//     //         name: req.params.search,
+//     //         address,
+//     //         img,
+
+//     //     })
+//     // }
+
+const search = async (req, res, next) => {
+    try {
         let reviews = []
-            $1('.kCrYT').each(function (i) {
-                let link = ""
-                if ($1(this).find('a').attr('href')) {
-                    link = ($1(this).find('a').attr('href')).match(editUrl)
-                }
-                reviews[i] = {
-                    title: $1(this).find('h3').text(),
-                    link: link,
-                    summary: $1(this).find('.s3v9rd').text(),
+        let content = []
+        const url = "https://www.google.com/search?q=" + encodeURI(req.params.search) + "+site%3Atistory.com&page_no=1"
+        const response1 = await axios.get(url)
+        // console.log(response1.data)
+        const $1 = cheerio.load(response1.data);
+        $1('.kCrYT').each(async function (i) {
+            const title = $1(this).find('h3').text()
+            const searchParams = new URLSearchParams($1(this).find('a').attr('href'));
+            const link = searchParams.get("/url?q")
+            const summary = $1(this).find('.s3v9rd').find('.s3v9rd').text()
+            content.push(getReview(link))
+            if (title) {
+                reviews[i] = { title: title, link: link }
+            } else if (summary) {
+                reviews[i - 1] = { ...reviews[i - 1], summary: summary }
+                reviews = reviews.filter(e => e)
             }
         })
-            // reviews.forEach((review, i) => {
-            //     axios.get(review.link)
-            //         .then((html) => {
-            //             let $2 = cheerio.load(html.data);
-            //             $2('').each(function(i){
-            //                 review.content($2(this).find(''.text()))
-            //             })
-            //         })
-            // })
-
-            console.log(reviews)
-            res.send(reviews)
+        const promiseReview = await Promise.all(content)
+        reviews.forEach(async(review, i) => {
+            review["content"] = promiseReview[i]
         })
-    // function anyToUtf8(str) {
-    //     const { encoding } = jschardet.detect(str); // 웹페이지 문서의 인코딩 타입을 확인
-    //     const iconv = new Iconv(encoding, "utf-8//translit//ignore"); // euc-kr 인코딩변환
-    //     return iconv.convert(str).toString();
-    // }
-    // try {
-    //     const newPlace = await new Place({
-    //         name: req.params.search,
-    //         address,
-    //         img,
-
-    //     })
-    // }
+        res.send(reviews)
+    } catch (error) {
+        console.log(error)
+        res.send(error)
+    }
 }
 
+const getReview = async (link) => {
+    let content = '없음'
+    if (link) {
+        const res = await axios.get(link)
+        const $2 = cheerio.load(res.data);
+        if ($2('.tt_article_useless_p_margin').text()) {
+            content = $2('.tt_article_useless_p_margin').text()
+        }
+    }
+    return content
+}
 
-export default { search }
+export default { search, getReview }
\ No newline at end of file

server/models/Review.js

@@ -8,20 +8,23 @@ const ReviewSchema = new mongoose.Schema({
     },
     link: {
         type: String,
-        required: true,
+        // required: true,
         unique: true,
     },
     summary: {
         type: String,
-        required: true,
+        // required: true,
+    },
+    content: {
+        type: String,
     },
     keyword: {
         type: Array,
-        required: true,
+        // required: true,
     },
     time: {
         type: Array,
-        required: true,
+        // required: true,
     },
 }, {
     timestamps: true

server/scraper.js

-import axios from "axios";
-import cheerio from "cheerio";
-import express from 'express';
-import request from 'request-promise'
-import jschardet from 'jschardet'
-import iconv from 'iconv'
-import fs from 'fs'
-const Iconv = iconv.Iconv
-
-const app = express()
-
-app.get('/', (req, res) => {
-    const url = "https://100mountain.tistory.com/117"
-    request({
-        url: url,
-        encoding: null,
-    })
-        .then(anyToUtf8)
-        .then((html) => {
-            // fs.writeFileSync("test.txt", '\ufeff' + html, {encoding: 'utf8'});
-            let $ = cheerio.load(html, null, false);
-            let places = []
-
-            $('div.tt_article_useless_p_margin').each(function () {
-                console.log("title", $(this).find('p').text())
-            });
-            console.log("places", places)
-        })
-    function anyToUtf8(str) {
-        const { encoding } = jschardet.detect(str);
-        const iconv = new Iconv(encoding, "utf-8//translit//ignore");
-        return iconv.convert(str).toString();
-    }
-})
-
-app.listen(3001, () => {
-    console.log('Server is listening on port 3001')
-})
\ No newline at end of file

server/scraperss.js

-import axios from "axios";
-import cheerio from "cheerio";
-import express from 'express';
-import request from 'request-promise'
-import jschardet from 'jschardet'
-import iconv from 'iconv'
-import fs from 'fs'
-const Iconv = iconv.Iconv
-
-const app = express()
-
-app.get('/', (req, res) => {
-    const url = "https://www.google.com/search?q=%ED%95%9C%EB%9D%BC%EC%82%B0%20site%3Atistory.com&oq=tistory&aqs=chrome..69i57j0l4j69i60l3.1746j0j4&sourceid=chrome&ie=UTF-8&ved=2ahUKEwis_bSFz4buAhWVdXAKHU0tBaoQ2wF6BAgIEAE&ei=T1D1X-yZD5XrwQPN2pTQCg"
-
-    request(url)
-        .then(anyToUtf8)
-        .then((html) => {
-            // fs.writeFileSync("googlez.txt", '\ufeff' + html, { encoding: 'utf8' });
-
-            let $ = cheerio.load(html, null, false);
-            let places = []
-            $('.kCrYT').each(function (i) {
-                places[i] = {
-                    title: $(this).find('h3').text(),
-                    link: $(this).find('a').attr('href'),
-                    summary: $(this).text(),
-                }
-            })
-            console.log(places)
-        })
-    function anyToUtf8(str) {
-        const { encoding } = jschardet.detect(str);
-        console.log("source encoding = " + encoding);
-        const iconv = new Iconv(encoding, "utf-8//translit//ignore");
-        return iconv.convert(str).toString();
-    }
-    res.send("안녕")
-})
-
-app.listen(3001, () => {
-    console.log('Server is listening on port 3001')
-})
\ No newline at end of file