From 7a24f7284936a8e10c82730fa2bdb1083df51b89 Mon Sep 17 00:00:00 2001 From: Ruben Bimberg <ruben.bimberg@mni.thm.de> Date: Sat, 19 Jun 2021 17:06:22 +0200 Subject: [PATCH] Update refresh worker for spacy keywords --- .../create-comment.component.ts | 62 +++++----- .../worker-dialog/worker-dialog-task.ts | 93 +++++++++++++++ .../worker-dialog.component.html | 9 +- .../worker-dialog/worker-dialog.component.ts | 112 ++++++++---------- .../comment-list/comment-list.component.ts | 6 +- .../shared/header/header.component.ts | 37 +----- src/app/services/http/languagetool.service.ts | 74 ++++++++++-- src/app/utils/create-comment-keywords.ts | 27 +++++ 8 files changed, 273 insertions(+), 147 deletions(-) create mode 100644 src/app/components/shared/_dialogs/worker-dialog/worker-dialog-task.ts create mode 100644 src/app/utils/create-comment-keywords.ts diff --git a/src/app/components/shared/_dialogs/create-comment/create-comment.component.ts b/src/app/components/shared/_dialogs/create-comment/create-comment.component.ts index 220f59a30..206b884cc 100644 --- a/src/app/components/shared/_dialogs/create-comment/create-comment.component.ts +++ b/src/app/components/shared/_dialogs/create-comment/create-comment.component.ts @@ -9,6 +9,7 @@ import { CommentListComponent } from '../../comment-list/comment-list.component' import { EventService } from '../../../../services/util/event.service'; import { SpacyDialogComponent } from '../spacy-dialog/spacy-dialog.component'; import { LanguagetoolService, Language } from '../../../../services/http/languagetool.service'; +import { CreateCommentKeywords } from '../../../../utils/create-comment-keywords'; @Component({ selector: 'app-submit-comment', @@ -75,7 +76,7 @@ export class CreateCommentComponent implements OnInit, OnDestroy { this.dialogRef.close(); } - onPaste(e){ + onPaste(e) { e.preventDefault(); const elem = document.getElementById('answer-input'); const text = e.clipboardData.getData('text'); @@ -120,34 +121,27 @@ export class CreateCommentComponent implements OnInit, OnDestroy { } openSpacyDialog(comment: Comment): void { - const filteredInputText = this.checkUTFEmoji(this.inputText); - this.checkSpellings(filteredInputText).subscribe((res) => { - const words: string[] = filteredInputText.trim().split(' '); - const errorQuotient = (res.matches.length * 100) / words.length; - const hasSpellcheckConfidence = this.checkLanguageConfidence(res); - - if (hasSpellcheckConfidence && errorQuotient <= 20) { - const commentLang = this.languagetoolService.mapLanguageToSpacyModel(res.language.code); - - const dialogRef = this.dialog.open(SpacyDialogComponent, { - data: { - comment, - commentLang, - commentBodyChecked: filteredInputText - } - }); - - dialogRef.afterClosed() - .subscribe(result => { - if (result) { - this.dialogRef.close(result); + CreateCommentKeywords.isSpellingAcceptable(this.languagetoolService, this.inputText, this.selectedLang) + .subscribe((result) => { + if (result.isAcceptable) { + const commentLang = this.languagetoolService.mapLanguageToSpacyModel(result.result.language.code as Language); + const dialogRef = this.dialog.open(SpacyDialogComponent, { + data: { + comment, + commentLang, + commentBodyChecked: result.text } }); - } else { - this.dialogRef.close(comment); - } - }); - }; + dialogRef.afterClosed().subscribe(dialogResult => { + if (dialogResult) { + this.dialogRef.close(dialogResult); + } + }); + } else { + this.dialogRef.close(comment); + } + }); + } /** * Returns a lambda which closes the dialog on call. @@ -190,15 +184,15 @@ export class CreateCommentComponent implements OnInit, OnDestroy { this.hasSpellcheckConfidence = false; return; } - if(this.selectedLang === 'auto' && (document.getElementById('langSelect').innerText.includes(this.newLang) + if (this.selectedLang === 'auto' && (document.getElementById('langSelect').innerText.includes(this.newLang) || document.getElementById('langSelect').innerText.includes('auto'))) { - if(wordsCheck.language.name.includes('German')){ + if (wordsCheck.language.name.includes('German')) { this.selectedLang = 'de-DE'; - }else if(wordsCheck.language.name.includes('English')){ - this.selectedLang= 'en-US'; - }else if(wordsCheck.language.name.includes('French')){ + } else if (wordsCheck.language.name.includes('English')) { + this.selectedLang = 'en-US'; + } else if (wordsCheck.language.name.includes('French')) { this.selectedLang = 'fr'; - }else{ + } else { this.newLang = wordsCheck.language.name; } document.getElementById('langSelect').innerHTML = this.newLang; @@ -278,7 +272,7 @@ export class CreateCommentComponent implements OnInit, OnDestroy { }, 500); }); } - }, () => {}, () => { + }, () => '', () => { this.isSpellchecking = false; }); } diff --git a/src/app/components/shared/_dialogs/worker-dialog/worker-dialog-task.ts b/src/app/components/shared/_dialogs/worker-dialog/worker-dialog-task.ts new file mode 100644 index 000000000..4a9613780 --- /dev/null +++ b/src/app/components/shared/_dialogs/worker-dialog/worker-dialog-task.ts @@ -0,0 +1,93 @@ +import { Room } from '../../../../models/room'; +import { Model, SpacyService } from '../../../../services/http/spacy.service'; +import { CommentService } from '../../../../services/http/comment.service'; +import { Comment } from '../../../../models/comment'; +import { Language, LanguagetoolService } from '../../../../services/http/languagetool.service'; +import { CreateCommentKeywords } from '../../../../utils/create-comment-keywords'; +import { TSMap } from 'typescript-map'; +import { HttpErrorResponse } from '@angular/common/http'; + +const concurrentCallsPerTask = 4; + +export class WorkerDialogTask { + + initializing = true; + error: string = null; + readonly statistics = { + succeeded: 0, + badSpelled: 0, + failed: 0, + length: 0 + }; + private _comments: Comment[] = null; + private _running: boolean[] = null; + + constructor(public readonly room: Room, + private spacyService: SpacyService, + private commentService: CommentService, + private languagetoolService: LanguagetoolService, + private finished: () => void) { + this.commentService.getAckComments(room.id).subscribe((c) => { + this._comments = c; + this.statistics.length = c.length; + this.initializing = false; + this._running = new Array(concurrentCallsPerTask); + for (let i = 0; i < concurrentCallsPerTask; i++) { + this._running[i] = true; + this.callSpacy(i); + } + }); + } + + private callSpacy(currentIndex: number) { + if (this.error || currentIndex >= this._comments.length) { + this._running[currentIndex % concurrentCallsPerTask] = false; + if (this._running.every(e => e === false)) { + if (this.finished) { + this.finished(); + this.finished = null; + } + } + return; + } + const fallbackmodel = (localStorage.getItem('currentLang') || 'de') as Model; + const currentComment = this._comments[currentIndex]; + CreateCommentKeywords.isSpellingAcceptable(this.languagetoolService, currentComment.body) + .subscribe(result => { + if (!result.isAcceptable) { + this.statistics.badSpelled++; + this.callSpacy(currentIndex + concurrentCallsPerTask); + return; + } + const model = this.languagetoolService + .mapLanguageToSpacyModel(result.result.language.detectedLanguage.code as Language); + this.spacyService.getKeywords(result.text, model === 'auto' ? fallbackmodel : model) + .subscribe(newKeywords => { + const changes = new TSMap<string, string>(); + changes.set('keywordsFromSpacy', JSON.stringify(newKeywords)); + this.commentService.patchComment(currentComment, changes).subscribe(_ => { + this.statistics.succeeded++; + }, + patchError => { + this.statistics.failed++; + if (patchError instanceof HttpErrorResponse && patchError.status === 403) { + this.error = 'forbidden'; + } + console.log(patchError); + }, () => { + this.callSpacy(currentIndex + concurrentCallsPerTask); + }); + }, + keywordError => { + this.statistics.failed++; + console.log(keywordError); + this.callSpacy(currentIndex + concurrentCallsPerTask); + }); + }, error => { + this.statistics.failed++; + console.log(error); + this.callSpacy(currentIndex + concurrentCallsPerTask); + }); + } + +} diff --git a/src/app/components/shared/_dialogs/worker-dialog/worker-dialog.component.html b/src/app/components/shared/_dialogs/worker-dialog/worker-dialog.component.html index 5872cbd52..a57273c93 100644 --- a/src/app/components/shared/_dialogs/worker-dialog/worker-dialog.component.html +++ b/src/app/components/shared/_dialogs/worker-dialog/worker-dialog.component.html @@ -1,18 +1,17 @@ <div id="worker-content"> - <div id="header"> - + <div id="header" (window:beforeunload)="checkTasks($event)"> <details> <summary> - <span>{{'worker-dialog.running' | translate}} # {{getNumberInQueue()}}</span> + <span>{{'worker-dialog.running' | translate}} # {{getRooms().length}}</span> <span><button id="btn_hide" (click)="close()">x</button></span> </summary> <div mat-dialog-content> - <div id="entry" *ngFor="let task of taskQueue"> + <div id="entry" *ngFor="let task of getRooms().values()"> <mat-icon svgIcon="meeting_room"></mat-icon> <span>{{ task.room.name }}</span> <span style="width: 10px"></span> <mat-icon>comment</mat-icon> - <span>{{ task.comments.length }}</span> + <span>{{ task.statistics.length }}</span> </div> </div> </details> diff --git a/src/app/components/shared/_dialogs/worker-dialog/worker-dialog.component.ts b/src/app/components/shared/_dialogs/worker-dialog/worker-dialog.component.ts index 942e0f81b..bf1621a61 100644 --- a/src/app/components/shared/_dialogs/worker-dialog/worker-dialog.component.ts +++ b/src/app/components/shared/_dialogs/worker-dialog/worker-dialog.component.ts @@ -1,14 +1,11 @@ import { Component, OnInit } from '@angular/core'; import { Room } from '../../../../models/room'; import { CommentService } from '../../../../services/http/comment.service'; -import { Comment } from '../../../../models/comment'; -import {Model, SpacyService} from '../../../../services/http/spacy.service'; +import { SpacyService } from '../../../../services/http/spacy.service'; import { TSMap } from 'typescript-map'; - -export interface WorkTask { - room: Room; - comments: Comment[]; -} +import { MatDialog, MatDialogRef } from '@angular/material/dialog'; +import { WorkerDialogTask } from './worker-dialog-task'; +import { LanguagetoolService } from '../../../../services/http/languagetool.service'; @Component({ selector: 'app-worker-dialog', @@ -17,81 +14,70 @@ export interface WorkTask { }) export class WorkerDialogComponent implements OnInit { - isRunning = false; - taskQueue: WorkTask[] = []; - closeCallback: any = null; + private static dialogRef: MatDialogRef<WorkerDialogComponent> = null; + private static queuedRooms = new TSMap<string, WorkerDialogTask>(); constructor(private commentService: CommentService, + private languagetoolService: LanguagetoolService, private spacyService: SpacyService) { } - ngOnInit(): void { - } - - _callNextInQueue(): void { - if (!this.isQueueEmpty()) { - this.isRunning = true; - const task = this.taskQueue[0]; - this.runWorkTask(task); - } else { - this.isRunning = false; - setTimeout(() => this.close(), 2000); + static addWorkTask(dialog: MatDialog, room: Room): boolean { + if (!this.dialogRef) { + this.dialogRef = dialog.open(WorkerDialogComponent, { + width: '200px', + disableClose: true, + autoFocus: false, + position: {left: '50px', bottom: '50px'}, + role: 'dialog', + hasBackdrop: false, + closeOnNavigation: false, + panelClass: 'workerContainer' + }); + this.dialogRef.beforeClosed().subscribe(_ => { + for (const value of WorkerDialogComponent.queuedRooms.values()) { + value.error = 'interrupt'; + } + WorkerDialogComponent.queuedRooms.clear(); + }); } - } - - addWorkTask(room: Room): void { - if (this.taskQueue.find((t: WorkTask) => t.room.id === room.id)) { - return; + if (this.queuedRooms.has(room.id)) { + return false; } - - this.commentService.getAckComments(room.id).subscribe((comments: Comment[]) => { - const task: WorkTask = {room, comments}; - - this.taskQueue.push(task); - - if (!this.isRunning) { - this._callNextInQueue(); - } - }); + this.dialogRef.componentInstance.appendRoom(room); + return true; } - runWorkTask(task: WorkTask): void { - task.comments.forEach((c: Comment) => { - const model = (localStorage.getItem('currentLang') || 'de') as Model; - const text = c.body; - this.spacyService.getKeywords(text, model).subscribe((keywords: string[]) => { - const changes = new TSMap<string, string>(); - changes.set('keywordsFromSpacy', JSON.stringify(keywords)); - this.taskQueue = this.taskQueue.slice(1, this.taskQueue.length); + ngOnInit(): void { + } - this.commentService.patchComment(c, changes).subscribe(_ => { - this._callNextInQueue(); - }, _ => { - this._callNextInQueue(); - }); - }); - }); + checkTasks(event: BeforeUnloadEvent) { + if (WorkerDialogComponent.queuedRooms.length > 0) { + event.preventDefault(); + event.returnValue = ''; + } } - getNumberInQueue() { - return this.taskQueue.length; + getRooms() { + return WorkerDialogComponent.queuedRooms; } - isQueueEmpty(): boolean { - return this.taskQueue.length === 0; + appendRoom(room: Room) { + WorkerDialogComponent.queuedRooms.set(room.id, + new WorkerDialogTask(room, this.spacyService, this.commentService, this.languagetoolService, () => { + if (WorkerDialogComponent.queuedRooms.length === 0) { + setTimeout(() => this.close(), 2000); + } + }) + ); } close(): void { - this.taskQueue = [] - this.isRunning = false; - if (this.closeCallback) { - this.closeCallback(); + if (WorkerDialogComponent.dialogRef) { + WorkerDialogComponent.dialogRef.close(); + WorkerDialogComponent.dialogRef = null; } } - getCloseCallback(callback: () => void): void { - this.closeCallback = callback; - } - } diff --git a/src/app/components/shared/comment-list/comment-list.component.ts b/src/app/components/shared/comment-list/comment-list.component.ts index 42d5f6344..f32841e42 100644 --- a/src/app/components/shared/comment-list/comment-list.component.ts +++ b/src/app/components/shared/comment-list/comment-list.component.ts @@ -4,7 +4,6 @@ import { CommentService } from '../../../services/http/comment.service'; import { TranslateService } from '@ngx-translate/core'; import { LanguageService } from '../../../services/util/language.service'; import { Message } from '@stomp/stompjs'; -import { CreateCommentComponent } from '../_dialogs/create-comment/create-comment.component'; import { MatDialog } from '@angular/material/dialog'; import { WsCommentServiceService } from '../../../services/websockets/ws-comment-service.service'; import { User } from '../../../models/user'; @@ -17,11 +16,10 @@ import { NotificationService } from '../../../services/util/notification.service import { CorrectWrong } from '../../../models/correct-wrong.enum'; import { LiveAnnouncer } from '@angular/cdk/a11y'; import { EventService } from '../../../services/util/event.service'; -import { Observable, Subscription } from 'rxjs'; +import { Subscription } from 'rxjs'; import { AppComponent } from '../../../app.component'; import { Router, ActivatedRoute } from '@angular/router'; import { AuthenticationService } from '../../../services/http/authentication.service'; -import { Title } from '@angular/platform-browser'; import { TitleService } from '../../../services/util/title.service'; import { ModeratorsComponent } from '../../creator/_dialogs/moderators/moderators.component'; import { TagsComponent } from '../../creator/_dialogs/tags/tags.component'; @@ -29,9 +27,7 @@ import { DeleteCommentsComponent } from '../../creator/_dialogs/delete-comments/ import { Export } from '../../../models/export'; import { BonusTokenService } from '../../../services/http/bonus-token.service'; import { ModeratorService } from '../../../services/http/moderator.service'; -import { TopicCloudFilterComponent } from '../_dialogs/topic-cloud-filter/topic-cloud-filter.component'; import { CommentFilterOptions } from '../../../utils/filter-options'; -import { isObjectBindingPattern } from 'typescript'; import { CreateCommentWrapper } from '../../../utils/CreateCommentWrapper'; export enum Period { diff --git a/src/app/components/shared/header/header.component.ts b/src/app/components/shared/header/header.component.ts index 9cbf857e9..b32b224c9 100644 --- a/src/app/components/shared/header/header.component.ts +++ b/src/app/components/shared/header/header.component.ts @@ -6,7 +6,7 @@ import { User } from '../../../models/user'; import { UserRole } from '../../../models/user-roles.enum'; import { Location } from '@angular/common'; import { TranslateService } from '@ngx-translate/core'; -import {_MatDialogBase, MAT_DIALOG_DEFAULT_OPTIONS, MatDialog, MatDialogRef} from '@angular/material/dialog'; +import { MatDialog } from '@angular/material/dialog'; import { LoginComponent } from '../login/login.component'; import { DeleteAccountComponent } from '../_dialogs/delete-account/delete-account.component'; import { UserService } from '../../../services/http/user.service'; @@ -24,7 +24,7 @@ import { TopicCloudFilterComponent } from '../_dialogs/topic-cloud-filter/topic- import { RoomService } from '../../../services/http/room.service'; import { Room } from '../../../models/room'; import { TagCloudMetaData } from '../../../services/util/tag-cloud-data.service'; -import {WorkerDialogComponent} from "../_dialogs/worker-dialog/worker-dialog.component"; +import { WorkerDialogComponent } from '../_dialogs/worker-dialog/worker-dialog.component'; @Component({ selector: 'app-header', @@ -39,11 +39,10 @@ export class HeaderComponent implements OnInit { isSafari = 'false'; moderationEnabled: boolean; motdState = false; - room : Room; + room: Room; commentsCountQuestions = 0; commentsCountUsers = 0; commentsCountKeywords = 0; - workerDialogRef: MatDialogRef<WorkerDialogComponent, null> = null; constructor(public location: Location, private authenticationService: AuthenticationService, @@ -325,33 +324,7 @@ export class HeaderComponent implements OnInit { } public startWorkerDialog() { - - if (this.workerDialogRef == null) { - - this.workerDialogRef = this.dialog.open(WorkerDialogComponent, { - width: '200px', - disableClose: true, - autoFocus: false, - position: {left: '50px', bottom: '50px'}, - role: 'dialog', - hasBackdrop: false, - closeOnNavigation: false, - panelClass: 'workerContainer' - }); - - const component: WorkerDialogComponent = this.workerDialogRef.componentInstance; - component.getCloseCallback(() => { - this.workerDialogRef.close(); - this.workerDialogRef = null; - }); - component.addWorkTask(this.room); - } else { - const component: WorkerDialogComponent = this.workerDialogRef.componentInstance; - component.addWorkTask(this.room); - } - - } - - + WorkerDialogComponent.addWorkTask(this.dialog, this.room); + } } diff --git a/src/app/services/http/languagetool.service.ts b/src/app/services/http/languagetool.service.ts index 145c3693d..0305c5811 100644 --- a/src/app/services/http/languagetool.service.ts +++ b/src/app/services/http/languagetool.service.ts @@ -5,7 +5,64 @@ import { catchError } from 'rxjs/operators'; import { Model } from './spacy.service'; import { Observable } from 'rxjs'; -export type Language = 'de-DE' | 'en-US' | 'fr' | 'auto'; +export type Language = 'de-DE' | 'en-US' | 'fr' | 'auto'; + +export interface LanguagetoolResult { + software: { + name: string; + version: string; + buildDate: string; + apiVersion: number; + status?: string; + premium?: boolean; + premiumHint?: string; + }; + language: { + name: string; + code: string; + detectedLanguage: { + name: string; + code: string; + confidence?: number; + }; + }; + matches: { + message: string; + shortMessage?: string; + offset: number; + length: number; + replacements: { + value?: string; + }[]; + context: { + text: string; + offset: number; + length: number; + }; + sentence: string; + rule?: { + id: string; + subId?: string; + description: string; + urls?: { + value?: string; + }[]; + issueType?: string; + category: { + id?: string; + name?: string; + }; + }; + contextForSureMatch?: number; + ignoreForIncompleteSentence?: boolean; + type?: { + typeName?: string; + }; + }[]; + warnings?: { + incompleteResults?: boolean; + }; +} @Injectable({ providedIn: 'root' @@ -29,13 +86,14 @@ export class LanguagetoolService extends BaseHttpService { } } - checkSpellings(text: string, language: Language): Observable<any> { + checkSpellings(text: string, language: Language): Observable<LanguagetoolResult> { const url = '/languagetool'; - return this.http.get(url, {params: { - text, language - }}) - .pipe( - catchError(this.handleError<any>('checkSpellings')) - ); + return this.http.get<LanguagetoolResult>(url, { + params: { + text, language + } + }).pipe( + catchError(this.handleError<any>('checkSpellings')) + ); } } diff --git a/src/app/utils/create-comment-keywords.ts b/src/app/utils/create-comment-keywords.ts new file mode 100644 index 000000000..ab2994e0f --- /dev/null +++ b/src/app/utils/create-comment-keywords.ts @@ -0,0 +1,27 @@ +import { Language, LanguagetoolService } from '../services/http/languagetool.service'; +import { map } from 'rxjs/operators'; + +export class CreateCommentKeywords { + + static isSpellingAcceptable(languagetoolService: LanguagetoolService, text: string, language: Language = 'auto') { + text = this.cleanUTFEmoji(text); + return languagetoolService.checkSpellings(text, language).pipe( + map(result => { + const wordCount = text.trim().split(' ').length; + const hasConfidence = language === 'auto' ? result.language.detectedLanguage.confidence >= 0.5 : true; + const hasLessMistakes = (result.matches.length * 100) / wordCount <= 20; + return { + isAcceptable: hasConfidence && hasLessMistakes, + text, + result + }; + }) + ); + } + + private static cleanUTFEmoji(text: string): string { + // eslint-disable-next-line max-len + const regex = /(?:\:.*?\:|[\u2700-\u27bf]|(?:\ud83c[\udde6-\uddff]){2}|[\ud800-\udbff][\udc00-\udfff]|[\u0023-\u0039]\ufe0f?\u20e3|\u3299|\u3297|\u303d|\u3030|\u24c2|\ud83c[\udd70-\udd71]|\ud83c[\udd7e-\udd7f]|\ud83c\udd8e|\ud83c[\udd91-\udd9a]|\ud83c[\udde6-\uddff]|\ud83c[\ude01-\ude02]|\ud83c\ude1a|\ud83c\ude2f|\ud83c[\ude32-\ude3a]|\ud83c[\ude50-\ude51]|\u203c|\u2049|[\u25aa-\u25ab]|\u25b6|\u25c0|[\u25fb-\u25fe]|\u00a9|\u00ae|\u2122|\u2139|\ud83c\udc04|[\u2600-\u26FF]|\u2b05|\u2b06|\u2b07|\u2b1b|\u2b1c|\u2b50|\u2b55|\u231a|\u231b|\u2328|\u23cf|[\u23e9-\u23f3]|[\u23f8-\u23fa]|\ud83c\udccf|\u2934|\u2935|[\u2190-\u21ff])/g; + return text.replace(regex, ''); + } +} -- GitLab