Skip to content

GitLab

You need to sign in or sign up before continuing.
Commit df06ba88 authored by baesangjune's avatar baesangjune
Browse files

Merge remote-tracking branch 'origin/cherry' into Sangjune

parents 63b59e7d 750a7171
Expand all Hide whitespace changes
Inline Side-by-side
client/.eslintcache
View file @ df06ba88
[{"C:\\Users\\노트북펜\\Desktop\\2021winter\\search-page\\client\\src\\index.js":"1","C:\\Users\\노트북펜\\Desktop\\2021winter\\search-page\\client\\src\\reportWebVitals.js":"2","C:\\Users\\노트북펜\\Desktop\\2021winter\\search-page\\client\\src\\Pages\\App.js":"3","C:\\Users\\노트북펜\\Desktop\\2021winter\\search-page\\client\\src\\Pages\\Search.js":"4","C:\\Users\\노트북펜\\Desktop\\2021winter\\search-page\\client\\src\\Components\\Place.js":"5","C:\\Users\\노트북펜\\Desktop\\2021winter\\search-page\\client\\src\\Components\\Paginations.js":"6"},{"size":1133,"mtime":1610931750245,"results":"7","hashOfConfig":"8"},{"size":375,"mtime":1610931750280,"results":"9","hashOfConfig":"8"},{"size":1553,"mtime":1610931750241,"results":"10","hashOfConfig":"8"},{"size":40446,"mtime":1610953780087,"results":"11","hashOfConfig":"8"},{"size":5363,"mtime":1610931750240,"results":"12","hashOfConfig":"8"},{"size":2199,"mtime":1610931750239,"results":"13","hashOfConfig":"8"},{"filePath":"14","messages":"15","errorCount":0,"warningCount":0,"fixableErrorCount":0,"fixableWarningCount":0},"1hwnshv",{"filePath":"16","messages":"17","errorCount":0,"warningCount":0,"fixableErrorCount":0,"fixableWarningCount":0},{"filePath":"18","messages":"19","errorCount":0,"warningCount":3,"fixableErrorCount":0,"fixableWarningCount":0,"source":null},{"filePath":"20","messages":"21","errorCount":0,"warningCount":4,"fixableErrorCount":0,"fixableWarningCount":0,"source":null},{"filePath":"22","messages":"23","errorCount":0,"warningCount":6,"fixableErrorCount":0,"fixableWarningCount":0,"source":"24","usedDeprecatedRules":"25"},{"filePath":"26","messages":"27","errorCount":0,"warningCount":0,"fixableErrorCount":0,"fixableWarningCount":0,"usedDeprecatedRules":"25"},"C:\\Users\\노트북펜\\Desktop\\2021winter\\search-page\\client\\src\\index.js",[],"C:\\Users\\노트북펜\\Desktop\\2021winter\\search-page\\client\\src\\reportWebVitals.js",[],"C:\\Users\\노트북펜\\Desktop\\2021winter\\search-page\\client\\src\\Pages\\App.js",["28","29","30"],"C:\\Users\\노트북펜\\Desktop\\2021winter\\search-page\\client\\src\\Pages\\Search.js",["31","32","33","34"],"C:\\Users\\노트북펜\\Desktop\\2021winter\\search-page\\client\\src\\Components\\Place.js",["35","36","37","38","39","40"],"import axios from 'axios';\r\nimport React, { useEffect, useState } from 'react';\r\nimport { Modal, Container, Row, Col, Button, Badge, Card, Accordion, Carousel } from 'react-bootstrap';\r\n\r\nfunction Place(props) {\r\n const [reviews, setReviews] = useState([])\r\n const getReview = () => {\r\n axios.get(`/api/review/${props.search.name}`)\r\n .then(res => {\r\n console.log(\"res.data=\", res.data)\r\n setReviews(res.data)\r\n })\r\n .catch(err => {\r\n console.log(err)\r\n })\r\n }\r\n\r\n useEffect(() => {\r\n getReview();\r\n }, [])\r\n\r\n\r\n return (\r\n <Modal {...props}\r\n size=\"xl\"\r\n keyboard=\"true\"\r\n variant=\"backdrop.in\"\r\n // show={lgShow}\r\n // onHide={() => setLgShow(false)}\r\n aria-labelledby=\"example-modal-sizes-title-lg\">\r\n <Modal.Header closeButton>\r\n <Modal.Title id=\"contained-modal-title-vcenter\" style={{ fontSize: '40px' }}>\r\n {props.index + 1}. {props.search.name}\r\n </Modal.Title>\r\n </Modal.Header>\r\n <Modal.Body className=\"show-grid\">\r\n <Container style={{ fontSize: '40px' }}>\r\n {reviews.map((review, index) => {\r\n return (\r\n <Row className=\"mt-4\">\r\n <a href={review.link}>{review.title}</a>\r\n <div>{review.summary}</div>\r\n </Row>\r\n )\r\n })}\r\n </Container>\r\n\r\n {/* <Accordion defaultActiveKey=\"0\">\r\n <Accordion.Toggle as={Button} variant=\"link\" eventKey=\"0\">\r\n <a className=\"mb-2\">다음의 블로그 보기</a>\r\n </Accordion.Toggle>\r\n <Accordion.Collapse eventKey=\"0\">\r\n <Card.Body><div>주도 전역을 지배하는 한라산은 남한에서 가장 높은 산으로 높이는 1,947.3m이다. 한라산이라는 이름은 산이 높아 산정에 서면 은하수를 잡아당길 수 있다는 뜻이며, 부악·원산·선산·두무악·영주산·부라산·혈망봉·여장군 등으로도 불려왔다.</div></Card.Body>\r\n </Accordion.Collapse>\r\n <Row>\r\n <Accordion.Toggle as={Button} variant=\"link\" eventKey=\"1\">\r\n <a className=\"mb-2\">네이버의 블로그 보기</a>\r\n </Accordion.Toggle>\r\n <Accordion.Collapse eventKey=\"1\">\r\n <Card.Body><div>한라산은 1966년 한라산 천연보호구역으로, 1970년 국립공원으로 지정되었다. 그리고 2002년에는 유네스코 생물권보전지역으로 지정되었으며 2007년에는 유네스코 세계자연유산으로 등재되었다. 2008년에는 물장오리오름 산정화구호 습지가 람사르 습지로 등록되어 보호 관리되고 있다</div></Card.Body>\r\n </Accordion.Collapse>\r\n </Row>\r\n <Row>\r\n <Accordion.Toggle as={Button} variant=\"link\" eventKey=\"2\">\r\n <a className=\"mb-2\">구글의 블로그 보기</a>\r\n </Accordion.Toggle>\r\n <Accordion.Collapse eventKey=\"2\">\r\n <Card.Body><div>한라산은 제주도에 있는 해발 1,947.06m, 면적 약 1,820km²의 화산으로, 제주도의 면적 대부분을 차지하고 있다. 정상에 백록담이라는 화산호가 있는데, 백록담이라는 이름은 흰 사슴이 물을 먹는 곳이라는 뜻에서 왔다고 전해진다.</div></Card.Body>\r\n </Accordion.Collapse>\r\n </Row>\r\n </Accordion>\r\n <Row style={{ color: 'white' }}>\r\n <a href=\"https://www.daum.net/\" className=\"mb-2\">다음</a><a href=\"https://www.naver.com/\" className=\"mb-2\">네이버</a><a href=\"https://www.google.com/\" className=\"mb-2\">구글</a>\r\n </Row>\r\n // </Container>\r\n <Carousel>\r\n <Carousel.Item interval={1000}>\r\n <h3>다음 블로그</h3>\r\n <div>주도 전역을 지배하는 한라산은 남한에서 가장 높은 산으로 높이는 1,947.3m이다. 한라산이라는 이름은 산이 높아 산정에 서면 은하수를 잡아당길 수 있다는 뜻이며, 부악·원산·선산·두무악·영주산·부라산·혈망봉·여장군 등으로도 불려왔다.</div>\r\n </Carousel.Item>\r\n <Carousel.Item interval={1000}>\r\n <h3>네이버 블로그</h3>\r\n <div>한라산은 1966년 한라산 천연보호구역으로, 1970년 국립공원으로 지정되었다. 그리고 2002년에는 유네스코 생물권보전지역으로 지정되었으며 2007년에는 유네스코 세계자연유산으로 등재되었다. 2008년에는 물장오리오름 산정화구호 습지가 람사르 습지로 등록되어 보호 관리되고 있다</div>\r\n </Carousel.Item>\r\n <Carousel.Item interval={1000}>\r\n <h3>구글 블로그</h3>\r\n <div>한라산은 제주도에 있는 해발 1,947.06m, 면적 약 1,820km²의 화산으로, 제주도의 면적 대부분을 차지하고 있다. 정상에 백록담이라는 화산호가 있는데, 백록담이라는 이름은 흰 사슴이 물을 먹는 곳이라는 뜻에서 왔다고 전해진다.</div>\r\n </Carousel.Item>\r\n </Carousel> */}\r\n </Modal.Body>\r\n <Modal.Footer>\r\n <Button block onClick={props.onHide}>Close</Button>\r\n </Modal.Footer>\r\n </Modal>\r\n );\r\n}\r\n\r\nexport default Place;\r\n",["41","42"],"C:\\Users\\노트북펜\\Desktop\\2021winter\\search-page\\client\\src\\Components\\Paginations.js",[],{"ruleId":"43","severity":1,"message":"44","line":1,"column":17,"nodeType":"45","messageId":"46","endLine":1,"endColumn":26},{"ruleId":"43","severity":1,"message":"47","line":2,"column":10,"nodeType":"45","messageId":"46","endLine":2,"endColumn":14},{"ruleId":"43","severity":1,"message":"48","line":4,"column":64,"nodeType":"45","messageId":"46","endLine":4,"endColumn":75},{"ruleId":"43","severity":1,"message":"49","line":5,"column":83,"nodeType":"45","messageId":"46","endLine":5,"endColumn":93},{"ruleId":"43","severity":1,"message":"50","line":15,"column":12,"nodeType":"45","messageId":"46","endLine":15,"endColumn":18},{"ruleId":"51","severity":1,"message":"52","line":51,"column":8,"nodeType":"53","endLine":51,"endColumn":10,"suggestions":"54"},{"ruleId":"55","severity":1,"message":"56","line":491,"column":35,"nodeType":"57","endLine":491,"endColumn":76},{"ruleId":"43","severity":1,"message":"58","line":3,"column":33,"nodeType":"45","messageId":"46","endLine":3,"endColumn":36},{"ruleId":"43","severity":1,"message":"59","line":3,"column":46,"nodeType":"45","messageId":"46","endLine":3,"endColumn":51},{"ruleId":"43","severity":1,"message":"60","line":3,"column":53,"nodeType":"45","messageId":"46","endLine":3,"endColumn":57},{"ruleId":"43","severity":1,"message":"61","line":3,"column":59,"nodeType":"45","messageId":"46","endLine":3,"endColumn":68},{"ruleId":"43","severity":1,"message":"62","line":3,"column":70,"nodeType":"45","messageId":"46","endLine":3,"endColumn":78},{"ruleId":"51","severity":1,"message":"63","line":20,"column":6,"nodeType":"53","endLine":20,"endColumn":8,"suggestions":"64"},{"ruleId":"65","replacedBy":"66"},{"ruleId":"67","replacedBy":"68"},"no-unused-vars","'useEffect' is defined but never used.","Identifier","unusedVar","'Link' is defined but never used.","'FormControl' is defined but never used.","'Pagination' is defined but never used.","'mobile' is assigned a value but never used.","react-hooks/exhaustive-deps","React Hook useEffect has missing dependencies: 'getImg' and 'getPlace'. Either include them or remove the dependency array.","ArrayExpression",["69"],"react/jsx-no-duplicate-props","No duplicate props allowed","JSXAttribute","'Col' is defined but never used.","'Badge' is defined but never used.","'Card' is defined but never used.","'Accordion' is defined but never used.","'Carousel' is defined but never used.","React Hook useEffect has a missing dependency: 'getReview'. Either include it or remove the dependency array.",["70"],"no-native-reassign",["71"],"no-negated-in-lhs",["72"],{"desc":"73","fix":"74"},{"desc":"75","fix":"76"},"no-global-assign","no-unsafe-negation","Update the dependencies array to be: [getImg, getPlace]",{"range":"77","text":"78"},"Update the dependencies array to be: [getReview]",{"range":"79","text":"80"},[1678,1680],"[getImg, getPlace]",[555,557],"[getReview]"] [{"C:\\Users\\노트북펜\\Desktop\\2021winter\\search-page\\client\\src\\index.js":"1","C:\\Users\\노트북펜\\Desktop\\2021winter\\search-page\\client\\src\\reportWebVitals.js":"2","C:\\Users\\노트북펜\\Desktop\\2021winter\\search-page\\client\\src\\Pages\\App.js":"3","C:\\Users\\노트북펜\\Desktop\\2021winter\\search-page\\client\\src\\Pages\\Search.js":"4","C:\\Users\\노트북펜\\Desktop\\2021winter\\search-page\\client\\src\\Components\\Place.js":"5","C:\\Users\\노트북펜\\Desktop\\2021winter\\search-page\\client\\src\\Components\\Paginations.js":"6"},{"size":1133,"mtime":1610956158273,"results":"7","hashOfConfig":"8"},{"size":375,"mtime":1610931750280,"results":"9","hashOfConfig":"8"},{"size":1553,"mtime":1610931750241,"results":"10","hashOfConfig":"8"},{"size":11584,"mtime":1610956268248,"results":"11","hashOfConfig":"8"},{"size":5413,"mtime":1610956277089,"results":"12","hashOfConfig":"8"},{"size":2199,"mtime":1610931750239,"results":"13","hashOfConfig":"8"},{"filePath":"14","messages":"15","errorCount":0,"warningCount":0,"fixableErrorCount":0,"fixableWarningCount":0},"1hwnshv",{"filePath":"16","messages":"17","errorCount":0,"warningCount":0,"fixableErrorCount":0,"fixableWarningCount":0},{"filePath":"18","messages":"19","errorCount":0,"warningCount":3,"fixableErrorCount":0,"fixableWarningCount":0,"source":null},{"filePath":"20","messages":"21","errorCount":0,"warningCount":4,"fixableErrorCount":0,"fixableWarningCount":0,"source":null},{"filePath":"22","messages":"23","errorCount":0,"warningCount":6,"fixableErrorCount":0,"fixableWarningCount":0,"source":null},{"filePath":"24","messages":"25","errorCount":0,"warningCount":0,"fixableErrorCount":0,"fixableWarningCount":0,"usedDeprecatedRules":"26"},"C:\\Users\\노트북펜\\Desktop\\2021winter\\search-page\\client\\src\\index.js",[],"C:\\Users\\노트북펜\\Desktop\\2021winter\\search-page\\client\\src\\reportWebVitals.js",[],"C:\\Users\\노트북펜\\Desktop\\2021winter\\search-page\\client\\src\\Pages\\App.js",["27","28","29"],"C:\\Users\\노트북펜\\Desktop\\2021winter\\search-page\\client\\src\\Pages\\Search.js",["30","31","32","33"],"C:\\Users\\노트북펜\\Desktop\\2021winter\\search-page\\client\\src\\Components\\Place.js",["34","35","36","37","38","39"],"C:\\Users\\노트북펜\\Desktop\\2021winter\\search-page\\client\\src\\Components\\Paginations.js",[],["40","41"],{"ruleId":"42","severity":1,"message":"43","line":1,"column":17,"nodeType":"44","messageId":"45","endLine":1,"endColumn":26},{"ruleId":"42","severity":1,"message":"46","line":2,"column":10,"nodeType":"44","messageId":"45","endLine":2,"endColumn":14},{"ruleId":"42","severity":1,"message":"47","line":4,"column":64,"nodeType":"44","messageId":"45","endLine":4,"endColumn":75},{"ruleId":"42","severity":1,"message":"48","line":5,"column":83,"nodeType":"44","messageId":"45","endLine":5,"endColumn":93},{"ruleId":"42","severity":1,"message":"49","line":15,"column":12,"nodeType":"44","messageId":"45","endLine":15,"endColumn":18},{"ruleId":"50","severity":1,"message":"51","line":51,"column":8,"nodeType":"52","endLine":51,"endColumn":10,"suggestions":"53"},{"ruleId":"54","severity":1,"message":"55","line":115,"column":35,"nodeType":"56","endLine":115,"endColumn":76},{"ruleId":"42","severity":1,"message":"57","line":3,"column":33,"nodeType":"44","messageId":"45","endLine":3,"endColumn":36},{"ruleId":"42","severity":1,"message":"58","line":3,"column":46,"nodeType":"44","messageId":"45","endLine":3,"endColumn":51},{"ruleId":"42","severity":1,"message":"59","line":3,"column":53,"nodeType":"44","messageId":"45","endLine":3,"endColumn":57},{"ruleId":"42","severity":1,"message":"60","line":3,"column":59,"nodeType":"44","messageId":"45","endLine":3,"endColumn":68},{"ruleId":"42","severity":1,"message":"61","line":3,"column":70,"nodeType":"44","messageId":"45","endLine":3,"endColumn":78},{"ruleId":"50","severity":1,"message":"62","line":20,"column":6,"nodeType":"52","endLine":20,"endColumn":8,"suggestions":"63"},{"ruleId":"64","replacedBy":"65"},{"ruleId":"66","replacedBy":"67"},"no-unused-vars","'useEffect' is defined but never used.","Identifier","unusedVar","'Link' is defined but never used.","'FormControl' is defined but never used.","'Pagination' is defined but never used.","'mobile' is assigned a value but never used.","react-hooks/exhaustive-deps","React Hook useEffect has missing dependencies: 'getImg' and 'getPlace'. Either include them or remove the dependency array.","ArrayExpression",["68"],"react/jsx-no-duplicate-props","No duplicate props allowed","JSXAttribute","'Col' is defined but never used.","'Badge' is defined but never used.","'Card' is defined but never used.","'Accordion' is defined but never used.","'Carousel' is defined but never used.","React Hook useEffect has a missing dependency: 'getReview'. Either include it or remove the dependency array.",["69"],"no-native-reassign",["70"],"no-negated-in-lhs",["71"],{"desc":"72","fix":"73"},{"desc":"74","fix":"75"},"no-global-assign","no-unsafe-negation","Update the dependencies array to be: [getImg, getPlace]",{"range":"76","text":"77"},"Update the dependencies array to be: [getReview]",{"range":"78","text":"79"},[1678,1680],"[getImg, getPlace]",[560,562],"[getReview]"]
\ No newline at end of file \ No newline at end of file
client/src/Components/Place.js
View file @ df06ba88
...@@ -7,7 +7,7 @@ function Place(props) { ...@@ -7,7 +7,7 @@ function Place(props) {
const getReview = () => { const getReview = () => {
axios.get(`/api/review/${props.search.name}`) axios.get(`/api/review/${props.search.name}`)
.then(res => { .then(res => {
console.log("res.data=", res.data) console.log("place res.data", res.data)
setReviews(res.data) setReviews(res.data)
}) })
.catch(err => { .catch(err => {
...@@ -40,6 +40,7 @@ function Place(props) { ...@@ -40,6 +40,7 @@ function Place(props) {
<Row className="mt-4"> <Row className="mt-4">
<a href={review.link}>{review.title}</a> <a href={review.link}>{review.title}</a>
<div>{review.summary}</div> <div>{review.summary}</div>
<div>{review.content}</div>
</Row> </Row>
) )
})} })}
......
client/src/Pages/Search.js
View file @ df06ba88
This diff is collapsed.
server/controllers/review.controller copy.js 0 → 100644
View file @ df06ba88
import Review from '../models/Review.js'
import cheerio, { html } from "cheerio";
import jschardet from 'jschardet'
import iconv from 'iconv'
import fs from 'fs'
import axios from 'axios';
import { nextTick } from 'process';
const Iconv = iconv.Iconv
const search = async (req, res, next) => {
// const url = "https://www.google.com/search?q=" + encodeURI(req.params.search) + "+site%3Atistory.com&page_no=1"
const url = "https://www.google.com/search?q=" + encodeURI("한라산") + "+site%3Atistory.com&page_no=1"
let reviews = []
try {
axios.get(url)
// .then(anyToUtf8)
.then((html) => {
const $1 = cheerio.load(html.data);
$1('.kCrYT').each(function (i) {
const searchParams = new URLSearchParams($1(this).find('a').attr('href'));
const title = $1(this).find('h3').text()
const link = searchParams.get("/url?q")
const summary = $1(this).find('.s3v9rd').find('.s3v9rd').text()
const response = axios.get(link)
console.log(response)
// const $2 = cheerio.load(toString(response.data));
let content = "없음"
// if ($2('.tt_article_useless_p_margin').text()) {
// content = $2('.tt_article_useless_p_margin').text()
// }
if (title) {
reviews[i] = { title: title, link: link, content: content }
} else if (summary) {
reviews[i - 1] = { ...reviews[i - 1], summary: summary }
reviews = reviews.filter(e => e)
}
// reviews.forEach((review, i) => {
// axios.get(review.link)
// .then((html) => {
// const $2 = cheerio.load(html.data);
// let content = '없음'
// if ($2('.tt_article_useless_p_margin').text()) {
// content = $2('.tt_article_useless_p_margin').text()
// }
// // fs.writeFileSync(`tistory${i}.txt`, '\ufeff' + html.data, { encoding: 'utf8' });
// // console.log(`${i}번째, ${review.content}`)
// review["content"] = content
// // console.log(review)
// })
// })
// // const review = new Review(reviews).save()
})
res.send(reviews)
})
} catch (error) {
console.log(error)
res.status(500).send('리뷰 저장 에러')
}
}
const getReview = (link) => {
let content = '없음'
const res = axios.get(link)
console.log(res)
// const $2 = cheerio.load(res.data);
// if ($2('.tt_article_useless_p_margin').text()) {
// content = $2('.tt_article_useless_p_margin').text()
// }
console.log(content)
return content
}
export default { search, getReview }
\ No newline at end of file
server/controllers/review.controller..js 0 → 100644
View file @ df06ba88
import Review from '../models/Review.js'
import cheerio, { html } from "cheerio";
import jschardet from 'jschardet'
import iconv from 'iconv'
import fs from 'fs'
import axios from 'axios';
import { nextTick } from 'process';
import { response } from 'express';
const Iconv = iconv.Iconv
const search = async (req, res, next) => {
const url = "https://www.google.com/search?q=" + encodeURI(req.params.search) + "+site%3Atistory.com&page_no=1"
let reviews = []
try {
axios.get(url)
.then((response1) => {
const $1 = cheerio.load(response1.data);
$1('.kCrYT').each(function (i) {
const searchParams = new URLSearchParams($1(this).find('a').attr('href'));
const title = $1(this).find('h3').text()
const link = searchParams.get("/url?q")
const summary = $1(this).find('.s3v9rd').find('.s3v9rd').text()
if (title) {
reviews[i] = { title: title, link: link, }
} else if (summary) {
reviews[i - 1] = { ...reviews[i - 1], summary: summary }
reviews = reviews.filter(e => e)
}
// const review = new Review(reviews).save()
})
res.send(reviews)
})
} catch (error) {
// reviews.forEach((review) => {
// axios.get(review.link)
// .then((response2) => {
// const $2 = cheerio.load(response2.data);
// let content = ''
// if ($2('.tt_article_useless_p_margin').text()) {
// content = $2('.tt_article_useless_p_margin').text()
// }
// review = { content: content }
// console.log("dkssud")
// })
// console.log("dkssud")
// })
// console.log(content)
res.status(500).send(error)
}
}
const getReview = async (link) => {
let content = '없음'
const res = await axios.get(link)
const $2 = cheerio.load(res.data);
if ($2('.tt_article_useless_p_margin').text()) {
content = $2('.tt_article_useless_p_margin').text()
}
console.log(content, "getReiview")
return content
}
export default { search, getReview }
\ No newline at end of file
server/controllers/review.controller.js
View file @ df06ba88
...@@ -3,30 +3,51 @@ import cheerio from "cheerio"; ...@@ -3,30 +3,51 @@ import cheerio from "cheerio";
import jschardet from 'jschardet' import jschardet from 'jschardet'
import fs from 'fs' import fs from 'fs'
import axios from 'axios'; import axios from 'axios';
import { nextTick } from 'process';
// const Iconv = iconv.Iconv
const search = async (req, res) => { const search = async (req, res, next) => {
const url = "https://www.google.com/search?q=" + encodeURI(req.params.search) + "+site%3Atistory.com&page_no=1" try {
const editUrl = /(http(s)?:\/\/)([a-z0-9\w]+\.*)+[a-z0-9]{2,4}/gi let reviews = []
axios.get(url) let content = []
.then((response) => { const url = "https://www.google.com/search?q=" + encodeURI(req.params.search) + "+site%3Atistory.com&page_no=1"
const html = response.data const response1 = await axios.get(url)
fs.writeFileSync("googlez.txt", html, { encoding: 'utf8' }); // console.log(response1.data)
let $1 = cheerio.load(html); const $1 = cheerio.load(response1.data);
let reviews = [] $1('.kCrYT').each(async function (i) {
$1('.kCrYT').each(function (i) { const title = $1(this).find('h3').text()
const searchParams = new URLSearchParams($1(this).find('a').attr('href'));
if ($1(this).find('h3').text()) { const link = searchParams.get("/url?q")
reviews[i] = { title: $1(this).find('h3').text(), link: ($1(this).find('a').attr('href')) } const summary = $1(this).find('.s3v9rd').find('.s3v9rd').text()
} else if ($1(this).find('.s3v9rd').find('.s3v9rd').text()) { content.push(getReview(link))
reviews[i - 1] = { ...reviews[i - 1], summary: $1(this).find('.s3v9rd').find('.s3v9rd').text() } if (title) {
reviews = reviews.filter(e => e) reviews[i] = { title: title, link: link }
} } else if (summary) {
}) reviews[i - 1] = { ...reviews[i - 1], summary: summary }
reviews = reviews.filter(e => e)
// console.log("*******************************************",reviews) }
res.send(reviews)
}) })
const promiseReview = await Promise.all(content)
reviews.forEach(async(review, i) => {
review["content"] = promiseReview[i]
})
res.send(reviews)
} catch (error) {
console.log(error)
res.send(error)
}
} }
const getReview = async (link) => {
let content = '없음'
if (link) {
const res = await axios.get(link)
const $2 = cheerio.load(res.data);
if ($2('.tt_article_useless_p_margin').text()) {
content = $2('.tt_article_useless_p_margin').text()
}
}
return content
}
export default { search } export default { search, getReview }
server/models/Review.js
View file @ df06ba88
...@@ -8,20 +8,23 @@ const ReviewSchema = new mongoose.Schema({ ...@@ -8,20 +8,23 @@ const ReviewSchema = new mongoose.Schema({
}, },
link: { link: {
type: String, type: String,
required: true, // required: true,
unique: true, unique: true,
}, },
summary: { summary: {
type: String, type: String,
required: true, // required: true,
},
content: {
type: String,
}, },
keyword: { keyword: {
type: Array, type: Array,
required: true, // required: true,
}, },
time: { time: {
type: Array, type: Array,
required: true, // required: true,
}, },
}, { }, {
timestamps: true timestamps: true
......
server/scraper.js deleted 100644 → 0
View file @ 63b59e7d
import axios from "axios";
import cheerio from "cheerio";
import express from 'express';
import request from 'request-promise'
import jschardet from 'jschardet'
import iconv from 'iconv'
import fs from 'fs'
const Iconv = iconv.Iconv
const app = express()
app.get('/', (req, res) => {
const url = "https://100mountain.tistory.com/117"
request({
url: url,
encoding: null,
})
.then(anyToUtf8)
.then((html) => {
// fs.writeFileSync("test.txt", '\ufeff' + html, {encoding: 'utf8'});
let $ = cheerio.load(html, null, false);
let places = []
$('div.tt_article_useless_p_margin').each(function () {
console.log("title", $(this).find('p').text())
});
console.log("places", places)
})
function anyToUtf8(str) {
const { encoding } = jschardet.detect(str);
const iconv = new Iconv(encoding, "utf-8//translit//ignore");
return iconv.convert(str).toString();
}
})
app.listen(3001, () => {
console.log('Server is listening on port 3001')
})
\ No newline at end of file
server/scraperss.js deleted 100644 → 0
View file @ 63b59e7d
import axios from "axios";
import cheerio from "cheerio";
import express from 'express';
import request from 'request-promise'
import jschardet from 'jschardet'
// import iconv from 'iconv'
// import fs from 'fs'
// const Iconv = iconv.Iconv
const app = express()
app.get('/', (req, res) => {
const url = "https://www.google.com/search?q=%ED%95%9C%EB%9D%BC%EC%82%B0%20site%3Atistory.com&oq=tistory&aqs=chrome..69i57j0l4j69i60l3.1746j0j4&sourceid=chrome&ie=UTF-8&ved=2ahUKEwis_bSFz4buAhWVdXAKHU0tBaoQ2wF6BAgIEAE&ei=T1D1X-yZD5XrwQPN2pTQCg"
request(url)
// .then(anyToUtf8)
.then((html) => {
// fs.writeFileSync("googlez.txt", '\ufeff' + html, { encoding: 'utf8' });
let $ = cheerio.load(html, null, false);
let places = []
$('.kCrYT').each(function (i) {
places[i] = {
title: $(this).find('h3').text(),
link: $(this).find('a').attr('href'),
summary: $(this).text(),
}
})
console.log(places)
})
function anyToUtf8(str) {
const { encoding } = jschardet.detect(str);
console.log("source encoding = " + encoding);
const iconv = new Iconv(encoding, "utf-8//translit//ignore");
return iconv.convert(str).toString();
}
res.send("안녕")
})
app.listen(3001, () => {
console.log('Server is listening on port 3001')
})
\ No newline at end of file
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment