Browse Source

fix long sites

main
matthew 2 years ago
parent
commit
82d41c31fe
  1. 3
      searching-front/app/core/components/WebsiteCard/styles.module.css
  2. 6
      searching-front/services/main.ts
  3. 2
      searching-front/services/modules/parser/index.ts
  4. 3
      searching-front/services/parser.ts

3
searching-front/app/core/components/WebsiteCard/styles.module.css

@ -24,6 +24,9 @@
text-decoration: none;
font-size: 14px;
color: var(--text_light_secondary);
max-width: 300px;
white-space: nowrap;
overflow: hidden;
}
.description {

6
searching-front/services/main.ts

@ -6,9 +6,13 @@ import parser from './parser'
const run = async()=>{
console.log('Start domain watcher')
console.time('watcher')
await domainWatcher();
console.log('Start parser')
console.timeEnd('watcher')
console.log('Start parser');
console.time('watcher');
await parser();
console.timeEnd('watcher');
}
const second = 1000;

2
searching-front/services/modules/parser/index.ts

@ -69,7 +69,7 @@ class Parser {
subPages,
}
} catch (e) {
console.log("Error", e)
console.log("Parse error ", url)
return SHOULD_NOT_PARSE
}
}

3
searching-front/services/parser.ts

@ -11,7 +11,8 @@ const findFirstNotIndexed = (subpages: SubPages = {}) => {
}
const indexWebsite = async (domain: string, path: string, subpages: SubPages = {}) => {
if (!subpages[path]) {
const subpagesLength = Object.keys(subpages).length;
if (!subpages[path] && subpagesLength < 100) {
const url = domain + path;
const parseInfo = await Parser.parseUrl(url)
subpages[path] = true

Loading…
Cancel
Save