Skip to content

GitLab

Commit c8d82d40 authored by Lee SeoYeon's avatar Lee SeoYeon
Browse files

Merge remote-tracking branch 'origin/cherry' into lsy

parents 4fb36f66 750a7171
Expand all Hide whitespace changes
Inline Side-by-side
client/src/Components/Place.js
View file @ c8d82d40
......@@ -7,6 +7,7 @@ function Place(props) {
const getReview = () => {
axios.get(`/api/review/${props.search.name}`)
.then(res => {
console.log("place res.data", res.data)
setReviews(res.data)
})
.catch(err => {
......@@ -38,6 +39,7 @@ function Place(props) {
<Row className="mt-4">
<a href={review.link}>{review.title}</a>
<div>{review.summary}</div>
<div>{review.content}</div>
</Row>
)
})}
......
client/src/Search.js
View file @ c8d82d40
This diff is collapsed.
server/controllers/review.controller copy.js 0 → 100644
View file @ c8d82d40
import Review from '../models/Review.js'
import cheerio, { html } from "cheerio";
import jschardet from 'jschardet'
import iconv from 'iconv'
import fs from 'fs'
import axios from 'axios';
import { nextTick } from 'process';
const Iconv = iconv.Iconv
const search = async (req, res, next) => {
// const url = "https://www.google.com/search?q=" + encodeURI(req.params.search) + "+site%3Atistory.com&page_no=1"
const url = "https://www.google.com/search?q=" + encodeURI("한라산") + "+site%3Atistory.com&page_no=1"
let reviews = []
try {
axios.get(url)
// .then(anyToUtf8)
.then((html) => {
const $1 = cheerio.load(html.data);
$1('.kCrYT').each(function (i) {
const searchParams = new URLSearchParams($1(this).find('a').attr('href'));
const title = $1(this).find('h3').text()
const link = searchParams.get("/url?q")
const summary = $1(this).find('.s3v9rd').find('.s3v9rd').text()
const response = axios.get(link)
console.log(response)
// const $2 = cheerio.load(toString(response.data));
let content = "없음"
// if ($2('.tt_article_useless_p_margin').text()) {
// content = $2('.tt_article_useless_p_margin').text()
// }
if (title) {
reviews[i] = { title: title, link: link, content: content }
} else if (summary) {
reviews[i - 1] = { ...reviews[i - 1], summary: summary }
reviews = reviews.filter(e => e)
}
// reviews.forEach((review, i) => {
// axios.get(review.link)
// .then((html) => {
// const $2 = cheerio.load(html.data);
// let content = '없음'
// if ($2('.tt_article_useless_p_margin').text()) {
// content = $2('.tt_article_useless_p_margin').text()
// }
// // fs.writeFileSync(`tistory${i}.txt`, '\ufeff' + html.data, { encoding: 'utf8' });
// // console.log(`${i}번째, ${review.content}`)
// review["content"] = content
// // console.log(review)
// })
// })
// // const review = new Review(reviews).save()
})
res.send(reviews)
})
} catch (error) {
console.log(error)
res.status(500).send('리뷰 저장 에러')
}
}
const getReview = (link) => {
let content = '없음'
const res = axios.get(link)
console.log(res)
// const $2 = cheerio.load(res.data);
// if ($2('.tt_article_useless_p_margin').text()) {
// content = $2('.tt_article_useless_p_margin').text()
// }
console.log(content)
return content
}
export default { search, getReview }
\ No newline at end of file
server/controllers/review.controller..js 0 → 100644
View file @ c8d82d40
import Review from '../models/Review.js'
import cheerio, { html } from "cheerio";
import jschardet from 'jschardet'
import iconv from 'iconv'
import fs from 'fs'
import axios from 'axios';
import { nextTick } from 'process';
import { response } from 'express';
const Iconv = iconv.Iconv
const search = async (req, res, next) => {
const url = "https://www.google.com/search?q=" + encodeURI(req.params.search) + "+site%3Atistory.com&page_no=1"
let reviews = []
try {
axios.get(url)
.then((response1) => {
const $1 = cheerio.load(response1.data);
$1('.kCrYT').each(function (i) {
const searchParams = new URLSearchParams($1(this).find('a').attr('href'));
const title = $1(this).find('h3').text()
const link = searchParams.get("/url?q")
const summary = $1(this).find('.s3v9rd').find('.s3v9rd').text()
if (title) {
reviews[i] = { title: title, link: link, }
} else if (summary) {
reviews[i - 1] = { ...reviews[i - 1], summary: summary }
reviews = reviews.filter(e => e)
}
// const review = new Review(reviews).save()
})
res.send(reviews)
})
} catch (error) {
// reviews.forEach((review) => {
// axios.get(review.link)
// .then((response2) => {
// const $2 = cheerio.load(response2.data);
// let content = ''
// if ($2('.tt_article_useless_p_margin').text()) {
// content = $2('.tt_article_useless_p_margin').text()
// }
// review = { content: content }
// console.log("dkssud")
// })
// console.log("dkssud")
// })
// console.log(content)
res.status(500).send(error)
}
}
const getReview = async (link) => {
let content = '없음'
const res = await axios.get(link)
const $2 = cheerio.load(res.data);
if ($2('.tt_article_useless_p_margin').text()) {
content = $2('.tt_article_useless_p_margin').text()
}
console.log(content, "getReiview")
return content
}
export default { search, getReview }
\ No newline at end of file
server/controllers/review.controller.js
View file @ c8d82d40
......@@ -6,53 +6,95 @@ import cheerio from "cheerio";
import axios from 'axios';
// const Iconv = iconv.Iconv
const search = async (req, res) => {
const url = "https://www.google.com/search?q=" + encodeURI(req.params.search) + "+site%3Atistory.com&page_no=1"
const editUrl = /(http(s)?:\/\/)([a-z0-9\w]+\.*)+[a-z0-9]{2,4}/gi
axios.get(url)
// .then(anyToUtf8)
.then((html) => {
// fs.writeFileSync("googlez.txt", '\ufeff' + html, { encoding: 'utf8' });
let $1 = cheerio.load(html.data);
let reviews = []
$1('.kCrYT').each(function (i) {
let link = ""
if ($1(this).find('a').attr('href')) {
link = ($1(this).find('a').attr('href')).match(editUrl)
}
reviews[i] = {
title: $1(this).find('h3').text(),
link: link,
summary: $1(this).find('.s3v9rd').text(),
}
})
// reviews.forEach((review, i) => {
// axios.get(review.link)
// .then((html) => {
// let $2 = cheerio.load(html.data);
// $2('').each(function(i){
// review.content($2(this).find(''.text()))
// })
// })
// })
// const search = async (req, res) => {
// const url = "https://www.google.com/search?q=" + encodeURI(req.params.search) + "+site%3Atistory.com&page_no=1"
// const editUrl = /(http(s)?:\/\/)([a-z0-9\w]+\.*)+[a-z0-9]{2,4}/gi
// axios.get(url)
// // .then(anyToUtf8)
// .then((html) => {
// // fs.writeFileSync("googlez.txt", '\ufeff' + html, { encoding: 'utf8' });
// let $1 = cheerio.load(html.data);
// let reviews = []
// $1('.kCrYT').each(function (i) {
// let link = ""
// if ($1(this).find('a').attr('href')) {
// link = ($1(this).find('a').attr('href')).match(editUrl)
// }
// reviews[i] = {
// title: $1(this).find('h3').text(),
// link: link,
// summary: $1(this).find('.s3v9rd').text(),
// }
// })
// // reviews.forEach((review, i) => {
// // axios.get(review.link)
// // .then((html) => {
// // let $2 = cheerio.load(html.data);
// // $2('').each(function(i){
// // review.content($2(this).find(''.text()))
// // })
// // })
// // })
console.log(reviews)
res.send(reviews)
})
// function anyToUtf8(str) {
// const { encoding } = jschardet.detect(str); // 웹페이지 문서의 인코딩 타입을 확인
// const iconv = new Iconv(encoding, "utf-8//translit//ignore"); // euc-kr 인코딩변환
// return iconv.convert(str).toString();
// }
// try {
// const newPlace = await new Place({
// name: req.params.search,
// address,
// img,
// console.log(reviews)
// res.send(reviews)
// })
// // function anyToUtf8(str) {
// // const { encoding } = jschardet.detect(str); // 웹페이지 문서의 인코딩 타입을 확인
// // const iconv = new Iconv(encoding, "utf-8//translit//ignore"); // euc-kr 인코딩변환
// // return iconv.convert(str).toString();
// // }
// // try {
// // const newPlace = await new Place({
// // name: req.params.search,
// // address,
// // img,
// // })
// // }
// })
// }
const search = async (req, res, next) => {
try {
let reviews = []
let content = []
const url = "https://www.google.com/search?q=" + encodeURI(req.params.search) + "+site%3Atistory.com&page_no=1"
const response1 = await axios.get(url)
// console.log(response1.data)
const $1 = cheerio.load(response1.data);
$1('.kCrYT').each(async function (i) {
const title = $1(this).find('h3').text()
const searchParams = new URLSearchParams($1(this).find('a').attr('href'));
const link = searchParams.get("/url?q")
const summary = $1(this).find('.s3v9rd').find('.s3v9rd').text()
content.push(getReview(link))
if (title) {
reviews[i] = { title: title, link: link }
} else if (summary) {
reviews[i - 1] = { ...reviews[i - 1], summary: summary }
reviews = reviews.filter(e => e)
}
})
const promiseReview = await Promise.all(content)
reviews.forEach(async(review, i) => {
review["content"] = promiseReview[i]
})
res.send(reviews)
} catch (error) {
console.log(error)
res.send(error)
}
}
const getReview = async (link) => {
let content = '없음'
if (link) {
const res = await axios.get(link)
const $2 = cheerio.load(res.data);
if ($2('.tt_article_useless_p_margin').text()) {
content = $2('.tt_article_useless_p_margin').text()
}
}
return content
}
export default { search }
\ No newline at end of file
export default { search, getReview }
\ No newline at end of file
server/models/Review.js
View file @ c8d82d40
......@@ -8,20 +8,23 @@ const ReviewSchema = new mongoose.Schema({
},
link: {
type: String,
required: true,
// required: true,
unique: true,
},
summary: {
type: String,
required: true,
// required: true,
},
content: {
type: String,
},
keyword: {
type: Array,
required: true,
// required: true,
},
time: {
type: Array,
required: true,
// required: true,
},
}, {
timestamps: true
......
server/scraper.js deleted 100644 → 0
View file @ 4fb36f66
import axios from "axios";
import cheerio from "cheerio";
import express from 'express';
import request from 'request-promise'
import jschardet from 'jschardet'
import iconv from 'iconv'
import fs from 'fs'
const Iconv = iconv.Iconv
const app = express()
app.get('/', (req, res) => {
const url = "https://100mountain.tistory.com/117"
request({
url: url,
encoding: null,
})
.then(anyToUtf8)
.then((html) => {
// fs.writeFileSync("test.txt", '\ufeff' + html, {encoding: 'utf8'});
let $ = cheerio.load(html, null, false);
let places = []
$('div.tt_article_useless_p_margin').each(function () {
console.log("title", $(this).find('p').text())
});
console.log("places", places)
})
function anyToUtf8(str) {
const { encoding } = jschardet.detect(str);
const iconv = new Iconv(encoding, "utf-8//translit//ignore");
return iconv.convert(str).toString();
}
})
app.listen(3001, () => {
console.log('Server is listening on port 3001')
})
\ No newline at end of file
server/scraperss.js deleted 100644 → 0
View file @ 4fb36f66
import axios from "axios";
import cheerio from "cheerio";
import express from 'express';
import request from 'request-promise'
import jschardet from 'jschardet'
import iconv from 'iconv'
import fs from 'fs'
const Iconv = iconv.Iconv
const app = express()
app.get('/', (req, res) => {
const url = "https://www.google.com/search?q=%ED%95%9C%EB%9D%BC%EC%82%B0%20site%3Atistory.com&oq=tistory&aqs=chrome..69i57j0l4j69i60l3.1746j0j4&sourceid=chrome&ie=UTF-8&ved=2ahUKEwis_bSFz4buAhWVdXAKHU0tBaoQ2wF6BAgIEAE&ei=T1D1X-yZD5XrwQPN2pTQCg"
request(url)
.then(anyToUtf8)
.then((html) => {
// fs.writeFileSync("googlez.txt", '\ufeff' + html, { encoding: 'utf8' });
let $ = cheerio.load(html, null, false);
let places = []
$('.kCrYT').each(function (i) {
places[i] = {
title: $(this).find('h3').text(),
link: $(this).find('a').attr('href'),
summary: $(this).text(),
}
})
console.log(places)
})
function anyToUtf8(str) {
const { encoding } = jschardet.detect(str);
console.log("source encoding = " + encoding);
const iconv = new Iconv(encoding, "utf-8//translit//ignore");
return iconv.convert(str).toString();
}
res.send("안녕")
})
app.listen(3001, () => {
console.log('Server is listening on port 3001')
})
\ No newline at end of file
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment