Skip to content

GitLab

Commit aaaf8613 authored by Kim, Chaerin's avatar Kim, Chaerin
Browse files

중간체크

parent 9168132d
Expand all Hide whitespace changes
Inline Side-by-side
client/.eslintcache
View file @ aaaf8613
[{"C:\\Users\\sag06\\Desktop\\project\\search-page\\client\\src\\Components\\Place.js":"1","C:\\Users\\sag06\\Desktop\\project\\search-page\\client\\src\\index.js":"2","C:\\Users\\sag06\\Desktop\\project\\search-page\\client\\src\\reportWebVitals.js":"3","C:\\Users\\sag06\\Desktop\\project\\search-page\\client\\src\\Components\\Paginations.js":"4","C:\\Users\\sag06\\Desktop\\project\\search-page\\client\\src\\Pages\\Search.js":"5","C:\\Users\\sag06\\Desktop\\project\\search-page\\client\\src\\Pages\\App.js":"6"},{"size":5323,"mtime":1610343158606,"results":"7","hashOfConfig":"8"},{"size":1004,"mtime":1610344592277,"results":"9","hashOfConfig":"8"},{"size":362,"mtime":1608689675665,"results":"10","hashOfConfig":"8"},{"size":2199,"mtime":1610325754517,"results":"11","hashOfConfig":"8"},{"size":38272,"mtime":1610344608591,"results":"12","hashOfConfig":"8"},{"size":1428,"mtime":1610344561934,"results":"13","hashOfConfig":"8"},{"filePath":"14","messages":"15","errorCount":0,"warningCount":6,"fixableErrorCount":0,"fixableWarningCount":0,"source":null},"1n20xnc",{"filePath":"16","messages":"17","errorCount":0,"warningCount":0,"fixableErrorCount":0,"fixableWarningCount":0},{"filePath":"18","messages":"19","errorCount":0,"warningCount":0,"fixableErrorCount":0,"fixableWarningCount":0},{"filePath":"20","messages":"21","errorCount":0,"warningCount":0,"fixableErrorCount":0,"fixableWarningCount":0},{"filePath":"22","messages":"23","errorCount":0,"warningCount":3,"fixableErrorCount":0,"fixableWarningCount":0,"source":null},{"filePath":"24","messages":"25","errorCount":0,"warningCount":3,"fixableErrorCount":0,"fixableWarningCount":0,"source":null},"C:\\Users\\sag06\\Desktop\\project\\search-page\\client\\src\\Components\\Place.js",["26","27","28","29","30","31"],"C:\\Users\\sag06\\Desktop\\project\\search-page\\client\\src\\index.js",[],"C:\\Users\\sag06\\Desktop\\project\\search-page\\client\\src\\reportWebVitals.js",[],"C:\\Users\\sag06\\Desktop\\project\\search-page\\client\\src\\Components\\Paginations.js",[],"C:\\Users\\sag06\\Desktop\\project\\search-page\\client\\src\\Pages\\Search.js",["32","33","34"],"C:\\Users\\sag06\\Desktop\\project\\search-page\\client\\src\\Pages\\App.js",["35","36","37"],{"ruleId":"38","severity":1,"message":"39","line":3,"column":33,"nodeType":"40","messageId":"41","endLine":3,"endColumn":36},{"ruleId":"38","severity":1,"message":"42","line":3,"column":46,"nodeType":"40","messageId":"41","endLine":3,"endColumn":51},{"ruleId":"38","severity":1,"message":"43","line":3,"column":53,"nodeType":"40","messageId":"41","endLine":3,"endColumn":57},{"ruleId":"38","severity":1,"message":"44","line":3,"column":59,"nodeType":"40","messageId":"41","endLine":3,"endColumn":68},{"ruleId":"38","severity":1,"message":"45","line":3,"column":70,"nodeType":"40","messageId":"41","endLine":3,"endColumn":78},{"ruleId":"46","severity":1,"message":"47","line":19,"column":6,"nodeType":"48","endLine":19,"endColumn":8,"suggestions":"49"},{"ruleId":"38","severity":1,"message":"50","line":5,"column":83,"nodeType":"40","messageId":"41","endLine":5,"endColumn":93},{"ruleId":"38","severity":1,"message":"51","line":14,"column":12,"nodeType":"40","messageId":"41","endLine":14,"endColumn":18},{"ruleId":"52","severity":1,"message":"53","line":458,"column":35,"nodeType":"54","endLine":458,"endColumn":76},{"ruleId":"38","severity":1,"message":"55","line":1,"column":17,"nodeType":"40","messageId":"41","endLine":1,"endColumn":26},{"ruleId":"38","severity":1,"message":"56","line":2,"column":10,"nodeType":"40","messageId":"41","endLine":2,"endColumn":14},{"ruleId":"38","severity":1,"message":"57","line":4,"column":26,"nodeType":"40","messageId":"41","endLine":4,"endColumn":30},"no-unused-vars","'Col' is defined but never used.","Identifier","unusedVar","'Badge' is defined but never used.","'Card' is defined but never used.","'Accordion' is defined but never used.","'Carousel' is defined but never used.","react-hooks/exhaustive-deps","React Hook useEffect has a missing dependency: 'getReview'. Either include it or remove the dependency array.","ArrayExpression",["58"],"'Pagination' is defined but never used.","'mobile' is assigned a value but never used.","react/jsx-no-duplicate-props","No duplicate props allowed","JSXAttribute","'useEffect' is defined but never used.","'Link' is defined but never used.","'Form' is defined but never used.",{"desc":"59","fix":"60"},"Update the dependencies array to be: [getReview]",{"range":"61","text":"62"},[511,513],"[getReview]"] [{"C:\\Users\\sag06\\Desktop\\project\\search-page\\client\\src\\reportWebVitals.js":"1"},{"size":362,"mtime":1608689675665,"results":"2","hashOfConfig":"3"},{"filePath":"4","messages":"5","errorCount":0,"warningCount":0,"fixableErrorCount":0,"fixableWarningCount":0},"1n20xnc","C:\\Users\\sag06\\Desktop\\project\\search-page\\client\\src\\reportWebVitals.js",[]]
\ No newline at end of file \ No newline at end of file
client/src/Components/Place.js
View file @ aaaf8613
...@@ -7,6 +7,7 @@ function Place(props) { ...@@ -7,6 +7,7 @@ function Place(props) {
const getReview = () => { const getReview = () => {
axios.get(`/api/review/${props.search.name}`) axios.get(`/api/review/${props.search.name}`)
.then(res => { .then(res => {
console.log("place res.data", res.data)
setReviews(res.data) setReviews(res.data)
}) })
.catch(err => { .catch(err => {
...@@ -38,6 +39,7 @@ function Place(props) { ...@@ -38,6 +39,7 @@ function Place(props) {
<Row className="mt-4"> <Row className="mt-4">
<a href={review.link}>{review.title}</a> <a href={review.link}>{review.title}</a>
<div>{review.summary}</div> <div>{review.summary}</div>
<div>{review.content}</div>
</Row> </Row>
) )
})} })}
......
client/src/Pages/Search.js
View file @ aaaf8613
This diff is collapsed.
server/controllers/review.controller copy.js 0 → 100644
View file @ aaaf8613
import Review from '../models/Review.js'
import cheerio, { html } from "cheerio";
import jschardet from 'jschardet'
import iconv from 'iconv'
import fs from 'fs'
import axios from 'axios';
import { nextTick } from 'process';
const Iconv = iconv.Iconv
const search = async (req, res, next) => {
// const url = "https://www.google.com/search?q=" + encodeURI(req.params.search) + "+site%3Atistory.com&page_no=1"
const url = "https://www.google.com/search?q=" + encodeURI("한라산") + "+site%3Atistory.com&page_no=1"
let reviews = []
try {
axios.get(url)
// .then(anyToUtf8)
.then((html) => {
const $1 = cheerio.load(html.data);
$1('.kCrYT').each(function (i) {
const searchParams = new URLSearchParams($1(this).find('a').attr('href'));
const title = $1(this).find('h3').text()
const link = searchParams.get("/url?q")
const summary = $1(this).find('.s3v9rd').find('.s3v9rd').text()
const response = axios.get(link)
console.log(response)
// const $2 = cheerio.load(toString(response.data));
let content = "없음"
// if ($2('.tt_article_useless_p_margin').text()) {
// content = $2('.tt_article_useless_p_margin').text()
// }
if (title) {
reviews[i] = { title: title, link: link, content: content }
} else if (summary) {
reviews[i - 1] = { ...reviews[i - 1], summary: summary }
reviews = reviews.filter(e => e)
}
// reviews.forEach((review, i) => {
// axios.get(review.link)
// .then((html) => {
// const $2 = cheerio.load(html.data);
// let content = '없음'
// if ($2('.tt_article_useless_p_margin').text()) {
// content = $2('.tt_article_useless_p_margin').text()
// }
// // fs.writeFileSync(`tistory${i}.txt`, '\ufeff' + html.data, { encoding: 'utf8' });
// // console.log(`${i}번째, ${review.content}`)
// review["content"] = content
// // console.log(review)
// })
// })
// // const review = new Review(reviews).save()
})
res.send(reviews)
})
} catch (error) {
console.log(error)
res.status(500).send('리뷰 저장 에러')
}
}
const getReview = (link) => {
let content = '없음'
const res = axios.get(link)
console.log(res)
// const $2 = cheerio.load(res.data);
// if ($2('.tt_article_useless_p_margin').text()) {
// content = $2('.tt_article_useless_p_margin').text()
// }
console.log(content)
return content
}
export default { search, getReview }
\ No newline at end of file
server/controllers/review.controller..js 0 → 100644
View file @ aaaf8613
import Review from '../models/Review.js'
import cheerio, { html } from "cheerio";
import jschardet from 'jschardet'
import iconv from 'iconv'
import fs from 'fs'
import axios from 'axios';
import { nextTick } from 'process';
import { response } from 'express';
const Iconv = iconv.Iconv
const search = async (req, res, next) => {
const url = "https://www.google.com/search?q=" + encodeURI(req.params.search) + "+site%3Atistory.com&page_no=1"
let reviews = []
try {
axios.get(url)
.then((response1) => {
const $1 = cheerio.load(response1.data);
$1('.kCrYT').each(function (i) {
const searchParams = new URLSearchParams($1(this).find('a').attr('href'));
const title = $1(this).find('h3').text()
const link = searchParams.get("/url?q")
const summary = $1(this).find('.s3v9rd').find('.s3v9rd').text()
if (title) {
reviews[i] = { title: title, link: link, }
} else if (summary) {
reviews[i - 1] = { ...reviews[i - 1], summary: summary }
reviews = reviews.filter(e => e)
}
// const review = new Review(reviews).save()
})
res.send(reviews)
})
} catch (error) {
// reviews.forEach((review) => {
// axios.get(review.link)
// .then((response2) => {
// const $2 = cheerio.load(response2.data);
// let content = ''
// if ($2('.tt_article_useless_p_margin').text()) {
// content = $2('.tt_article_useless_p_margin').text()
// }
// review = { content: content }
// console.log("dkssud")
// })
// console.log("dkssud")
// })
// console.log(content)
res.status(500).send(error)
}
}
const getReview = async (link) => {
let content = '없음'
const res = await axios.get(link)
const $2 = cheerio.load(res.data);
if ($2('.tt_article_useless_p_margin').text()) {
content = $2('.tt_article_useless_p_margin').text()
}
console.log(content, "getReiview")
return content
}
export default { search, getReview }
\ No newline at end of file
server/controllers/review.controller.js
View file @ aaaf8613
import Review from '../models/Review.js' import Review from '../models/Review.js'
import cheerio from "cheerio"; import cheerio, { html } from "cheerio";
import jschardet from 'jschardet' import jschardet from 'jschardet'
import iconv from 'iconv' import iconv from 'iconv'
import fs from 'fs' import fs from 'fs'
import axios from 'axios'; import axios from 'axios';
import { nextTick } from 'process';
const Iconv = iconv.Iconv const Iconv = iconv.Iconv
const search = async (req, res) => { const search = async (req, res, next) => {
const url = "https://www.google.com/search?q=" + encodeURI(req.params.search) + "+site%3Atistory.com&page_no=1" // const url = "https://www.google.com/search?q=" + encodeURI(req.params.search) + "+site%3Atistory.com&page_no=1"
const editUrl = /(http(s)?:\/\/)([a-z0-9\w]+\.*)+[a-z0-9]{2,4}/gi try {
axios.get(url) const url = "https://www.google.com/search?q=kakao+site%3Atistory.com&page_no=1"
// .then(anyToUtf8) let reviews = []
.then((html) => { const response1 = await axios.get(url)
// fs.writeFileSync("googlez.txt", '\ufeff' + html, { encoding: 'utf8' }); const $1 = cheerio.load(response1.data);
let $1 = cheerio.load(html.data); $1('.kCrYT').each(async function (i) {
let reviews = [] const searchParams = new URLSearchParams($1(this).find('a').attr('href'));
$1('.kCrYT').each(function (i) { const title = $1(this).find('h3').text()
let link = "" const link = searchParams.get("/url?q")
if ($1(this).find('a').attr('href')) { const summary = $1(this).find('.s3v9rd').find('.s3v9rd').text()
link = ($1(this).find('a').attr('href')).match(editUrl) const content = await getReview(link)
} if (title) {
reviews[i] = { reviews[i] = { title: title, link: link, content: content }
title: $1(this).find('h3').text(), } else if (summary) {
link: link, reviews[i - 1] = { ...reviews[i - 1], summary: summary }
summary: $1(this).find('.s3v9rd').text(), reviews = reviews.filter(e => e)
} }
}) // const review = new Review(reviews).save()
// reviews.forEach((review, i) => {
// axios.get(review.link)
// .then((html) => {
// let $2 = cheerio.load(html.data);
// $2('').each(function(i){
// review.content($2(this).find(''.text()))
// })
// })
// })
console.log(reviews)
res.send(reviews)
}) })
function anyToUtf8(str) { res.send(reviews)
const { encoding } = jschardet.detect(str); // 웹페이지 문서의 인코딩 타입을 확인 } catch (error) {
const iconv = new Iconv(encoding, "utf-8//translit//ignore"); // euc-kr 인코딩변환 console.log(error)
return iconv.convert(str).toString(); res.send(error)
} }
// try {
// const newPlace = await new Place({
// name: req.params.search,
// address,
// img,
// })
// }
} }
const getReview = async (link) => {
let content = '없음'
const res = await axios.get(link)
const $2 = cheerio.load(res.data);
if ($2('.tt_article_useless_p_margin').text()) {
content = $2('.tt_article_useless_p_margin').text()
}
// console.log(content,"getReiview")
return content
}
export default { search } export default { search, getReview }
\ No newline at end of file \ No newline at end of file
server/models/Review.js
View file @ aaaf8613
...@@ -8,20 +8,23 @@ const ReviewSchema = new mongoose.Schema({ ...@@ -8,20 +8,23 @@ const ReviewSchema = new mongoose.Schema({
}, },
link: { link: {
type: String, type: String,
required: true, // required: true,
unique: true, unique: true,
}, },
summary: { summary: {
type: String, type: String,
required: true, // required: true,
},
content: {
type: String,
}, },
keyword: { keyword: {
type: Array, type: Array,
required: true, // required: true,
}, },
time: { time: {
type: Array, type: Array,
required: true, // required: true,
}, },
}, { }, {
timestamps: true timestamps: true
......
server/scraper.js deleted 100644 → 0
View file @ 9168132d
import axios from "axios";
import cheerio from "cheerio";
import express from 'express';
import request from 'request-promise'
import jschardet from 'jschardet'
import iconv from 'iconv'
import fs from 'fs'
const Iconv = iconv.Iconv
const app = express()
app.get('/', (req, res) => {
const url = "https://100mountain.tistory.com/117"
request({
url: url,
encoding: null,
})
.then(anyToUtf8)
.then((html) => {
// fs.writeFileSync("test.txt", '\ufeff' + html, {encoding: 'utf8'});
let $ = cheerio.load(html, null, false);
let places = []
$('div.tt_article_useless_p_margin').each(function () {
console.log("title", $(this).find('p').text())
});
console.log("places", places)
})
function anyToUtf8(str) {
const { encoding } = jschardet.detect(str);
const iconv = new Iconv(encoding, "utf-8//translit//ignore");
return iconv.convert(str).toString();
}
})
app.listen(3001, () => {
console.log('Server is listening on port 3001')
})
\ No newline at end of file
server/scraperss.js deleted 100644 → 0
View file @ 9168132d
import axios from "axios";
import cheerio from "cheerio";
import express from 'express';
import request from 'request-promise'
import jschardet from 'jschardet'
import iconv from 'iconv'
import fs from 'fs'
const Iconv = iconv.Iconv
const app = express()
app.get('/', (req, res) => {
const url = "https://www.google.com/search?q=%ED%95%9C%EB%9D%BC%EC%82%B0%20site%3Atistory.com&oq=tistory&aqs=chrome..69i57j0l4j69i60l3.1746j0j4&sourceid=chrome&ie=UTF-8&ved=2ahUKEwis_bSFz4buAhWVdXAKHU0tBaoQ2wF6BAgIEAE&ei=T1D1X-yZD5XrwQPN2pTQCg"
request(url)
.then(anyToUtf8)
.then((html) => {
// fs.writeFileSync("googlez.txt", '\ufeff' + html, { encoding: 'utf8' });
let $ = cheerio.load(html, null, false);
let places = []
$('.kCrYT').each(function (i) {
places[i] = {
title: $(this).find('h3').text(),
link: $(this).find('a').attr('href'),
summary: $(this).text(),
}
})
console.log(places)
})
function anyToUtf8(str) {
const { encoding } = jschardet.detect(str);
console.log("source encoding = " + encoding);
const iconv = new Iconv(encoding, "utf-8//translit//ignore");
return iconv.convert(str).toString();
}
res.send("안녕")
})
app.listen(3001, () => {
console.log('Server is listening on port 3001')
})
\ No newline at end of file
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment