Browse Source

add elastic env

main
matthew 2 years ago
parent
commit
305deb76ed
  1. 3
      searching-front/services/modules/parser/index.ts
  2. 2
      searching-front/services/parser.ts

3
searching-front/services/modules/parser/index.ts

@ -28,10 +28,11 @@ class Parser {
constructor() {}
parseUrl = async (url: string) => {
try {
console.log('Send request via ton proxy ', url)
const { data, headers } = await axios.get(url,{
proxy: getTonProxy(),
})
console.log('Success response from ', url)
const contentType = headers["content-type"].toLocaleLowerCase()
console.log(contentType)
if (!contentType.startsWith('text/html')) {

2
searching-front/services/parser.ts

@ -11,6 +11,7 @@ const findFirstNotIndexed = (subpages: SubPages = {}) => {
}
const indexWebsite = async (domain: string, path: string, subpages: SubPages = {}) => {
console.log('Start indexWebsite ', domain)
if (!subpages[path]) {
const url = domain + path;
const parseInfo = await Parser.parseUrl(url)
@ -46,6 +47,7 @@ const main = async () => {
console.log('Find domains', domains)
if (domains) {
for (const domain of domains) {
console.log('Update ', domain)
await db.nftDomain.update({
where: { address: domain.address },
data: { lastParse: new Date() },

Loading…
Cancel
Save