Merge branch '669-put-some-notorious-keywords-on-the-blacklist-for-all-rooms' into 'staging'

Resolve "Put some notorious keywords on the blacklist for all rooms"

Closes #659, #669, and #670

See merge request arsnova/frag.jetzt!685

src/app/components/shared/_dialogs/deep-ldialog/deep-ldialog.component.ts

@@ -8,6 +8,7 @@ import { ExplanationDialogComponent } from '../explanation-dialog/explanation-di
 import { DeepLService, FormalityType, TargetLang } from '../../../../services/http/deep-l.service';
 import { Observable } from 'rxjs';
 import { map } from 'rxjs/operators';
+import { CreateCommentKeywords } from '../../../../utils/create-comment-keywords';
 
 interface ResultValue {
   body: string;
@@ -48,7 +49,7 @@ export class DeepLDialogComponent implements OnInit, AfterViewInit {
     const delta = ViewCommentDataComponent.getDeltaFromData(body);
     const xml = delta.ops.reduce((acc, e, i) => {
       if (typeof e['insert'] === 'string' && e['insert'].trim().length) {
-        acc += '<x i="' + i + '">' + this.encodeHTML(e['insert']) + '</x>';
+        acc += '<x i="' + i + '">' + this.encodeHTML(CreateCommentKeywords.removeMarkdown(e['insert'])) + '</x>';
         e['insert'] = '';
       }
       return acc;

src/app/components/shared/tag-cloud/tag-cloud.component.ts

@@ -69,7 +69,9 @@ class TagComment implements CloudData {
 const transformationScaleKiller = /scale\([^)]*\)/;
 const transformationRotationKiller = /rotate\(([^)]*)\)/;
 
-const maskedCharsRegex = /[“”‘’„‚«»‹›『』﹃﹄「」﹁﹂",《》〈〉'`#&]|(\s(lu|li’u)(?=\s))|(^lu\s)|(\sli’u$)/gm;
+const maskedCharRegex = /[“”‘’„‚«»‹›『』﹃﹄「」﹁﹂",《》〈〉'`#&]|(\s(lu|li’u)(?=\s))|(^lu\s)|(\sli’u$)/;
+const httpRegex = /(https?:[^\s](\s|$))/;
+const hidelist = new RegExp(maskedCharRegex.source + '|' + httpRegex.source, 'gmi');
 
 const CONDITION_ROOM = 0;
 const CONDITION_BUILT = 1;
@@ -318,7 +320,7 @@ export class TagCloudComponent implements OnInit, OnDestroy, AfterContentInit {
           if (rotation === null || this._currentSettings.randomAngles) {
             rotation = Math.floor(Math.random() * 30 - 15);
           }
-          const filteredTag = tag.replace(maskedCharsRegex, '')
+          const filteredTag = tag.replace(hidelist, '')
             .replace(regexMaskKeyword, '').replace(/ +/, ' ').trim();
           newElements.push(new TagComment(filteredTag, tag, rotation, tagData.weight, tagData, newElements.length));
         }

src/app/services/util/topic-cloud-admin.service.ts

@@ -17,7 +17,11 @@ import { UserRole } from '../../models/user-roles.enum';
 import { CloudParameters } from '../../utils/cloud-parameters';
 import { RoomDataService } from './room-data.service';
 
-export const regexMaskKeyword = /\b(frage|antwort|aufgabe|hallo|test|bzw|muss|more to come)\b/gmi;
+const words = [
+  'frage', 'antwort', 'aufgabe', 'hallo', 'test', 'bzw', 'bzw.', 'muss', 'more to come', 'mal', 'zb', 'zb\\.',
+  'z\\.\\s*b\\.', 'zum beispiel', 'beispiel', 'jeder?', 'jede/r', 'bisschen', 'bißchen', 'okay', 'ok', 'o.k.'
+];
+export const regexMaskKeyword = new RegExp('\\b(' + words.join('|') + ')\\b|…|\\\\|\\/', 'gmi');
 
 @Injectable({
   providedIn: 'root',

src/app/utils/create-comment-keywords.ts

@@ -3,11 +3,17 @@ import { map } from 'rxjs/operators';
 
 export class CreateCommentKeywords {
 
-  static isKeywordAcceptable(keyword: string) {
+  static isKeywordAcceptable(keyword: string): boolean {
     const regex = /(^[ -@\[-`{-~]+$)/g;
     return keyword.match(regex) === null && keyword.length > 2;
   }
 
+  static removeMarkdown(text: string): string {
+    return text.replace(/([*_~]+(?=[^*_~\s]))|(^[ \t]*#+ )|(^[ \t]*>[> ]*)|(`+)/gm, '')
+      .replace(/([^*_~\s])[*_~]+/gm, '$1')
+      .replace(/\[([^\n\[\]]*)\]\(([^()\n]*)\)/gm, '$1 $2');
+  }
+
   static isSpellingAcceptable(languagetoolService: LanguagetoolService, text: string, language: Language = 'auto') {
     return languagetoolService.checkSpellings(text, language).pipe(
       map(result => {
@@ -16,10 +22,37 @@ export class CreateCommentKeywords {
         const hasLessMistakes = (result.matches.length * 100) / wordCount <= 20;
         return {
           isAcceptable: hasConfidence && hasLessMistakes,
-          text,
+          text: this.escapeForSpacy(text),
           result
         };
       })
     );
   }
+
+  private static escapeForSpacy(text: string): string {
+    text = this.makeCapslockLowercase(text);
+    return text.replace(/\(([^-\s)]+-)\)([^\s]+)/gmi, '$1$2');
+  }
+
+  private static makeCapslockLowercase(text: string): string {
+    const upperText = text.toUpperCase();
+    const regex = /\s+|$/gmi;
+    let m: RegExpExecArray;
+    let result = '';
+    let lastAddedIndex = 0;
+    while ((m = regex.exec(upperText)) !== null) {
+      const str = text.substring(lastAddedIndex, m.index);
+      if (m.index - lastAddedIndex >= 2 && str === upperText.substring(lastAddedIndex, m.index)) {
+        result += str.toLowerCase();
+      } else {
+        result += str;
+      }
+      result += text.substring(m.index, regex.lastIndex);
+      lastAddedIndex = regex.lastIndex;
+      if (regex.lastIndex === m.index) {
+        regex.lastIndex++;
+      }
+    }
+    return result;
+  }
 }