Commit 777a281f authored by Daniel Gerhardt's avatar Daniel Gerhardt

Merge branch 'improve-migrations' into 'master'

Improve migrations

See merge request arsnova/arsnova-backend!174
parents 486d7031 eb85c89d
......@@ -33,6 +33,7 @@ public class UserProfile extends Entity {
public enum AuthProvider {
NONE,
UNKNOWN,
ANONYMIZED,
ARSNOVA,
ARSNOVA_GUEST,
LDAP,
......
......@@ -57,6 +57,8 @@ public class FromV2Migrator {
static final String V2_TYPE_SCHOOL = "school";
static final String V2_TYPE_YESNO = "yesno";
static final String V2_TYPE_FREETEXT = "freetext";
static final String V2_TYPE_SLIDE = "slide";
static final String V2_TYPE_FLASHCARD = "flashcard";
static final String V2_TYPE_GRID = "grid";
private static final Map<String, de.thm.arsnova.model.Content.Format> formatMapping;
......@@ -71,6 +73,8 @@ public class FromV2Migrator {
formatMapping.put(V2_TYPE_SCHOOL, de.thm.arsnova.model.Content.Format.SCALE);
formatMapping.put(V2_TYPE_YESNO, de.thm.arsnova.model.Content.Format.BINARY);
formatMapping.put(V2_TYPE_FREETEXT, de.thm.arsnova.model.Content.Format.TEXT);
formatMapping.put(V2_TYPE_SLIDE, de.thm.arsnova.model.Content.Format.TEXT);
formatMapping.put(V2_TYPE_FLASHCARD, de.thm.arsnova.model.Content.Format.TEXT);
formatMapping.put(V2_TYPE_GRID, de.thm.arsnova.model.Content.Format.GRID);
}
......@@ -108,8 +112,7 @@ public class FromV2Migrator {
if (loggedIn != null) {
if (dbUser == null) {
copyCommonProperties(loggedIn, profile);
profile.setLoginId(loggedIn.getUser());
profile.setAuthProvider(detectAuthProvider(profile.getLoginId()));
updateProfileFromLoginId(profile, loggedIn.getUser());
profile.setCreationTimestamp(new Date());
}
profile.setLastLoginTimestamp(new Date(loggedIn.getTimestamp()));
......@@ -191,6 +194,8 @@ public class FromV2Migrator {
public de.thm.arsnova.model.Content migrate(final Content from) {
final de.thm.arsnova.model.Content to;
final Map<String, Map<String, Object>> extensions;
final Map<String, Object> v2;
switch (from.getQuestionType()) {
case V2_TYPE_ABCD:
case V2_TYPE_SC:
......@@ -217,6 +222,31 @@ public class FromV2Migrator {
case V2_TYPE_FREETEXT:
to = new de.thm.arsnova.model.Content();
to.setFormat(de.thm.arsnova.model.Content.Format.TEXT);
break;
case V2_TYPE_SLIDE:
to = new de.thm.arsnova.model.Content();
to.setFormat(de.thm.arsnova.model.Content.Format.TEXT);
extensions = new HashMap<>();
to.setExtensions(extensions);
v2 = new HashMap<>();
extensions.put("v2", v2);
v2.put("format", V2_TYPE_SLIDE);
break;
case V2_TYPE_FLASHCARD:
to = new de.thm.arsnova.model.Content();
to.setFormat(de.thm.arsnova.model.Content.Format.TEXT);
extensions = new HashMap<>();
to.setExtensions(extensions);
v2 = new HashMap<>();
extensions.put("v2", v2);
v2.put("format", V2_TYPE_FLASHCARD);
if (!from.getPossibleAnswers().isEmpty()) {
to.setAdditionalText(from.getPossibleAnswers().get(0).getText());
to.setAdditionalTextTitle("Back");
}
break;
default:
throw new IllegalArgumentException("Unsupported content format.");
......@@ -228,10 +258,10 @@ public class FromV2Migrator {
to.setBody(from.getText());
to.setAbstentionsAllowed(from.isAbstention());
to.setAbstentionsAllowed(from.isAbstention());
if (from.getSolution() != null) {
if (from.getSolution() != null && !from.getSolution().isEmpty()) {
to.setAdditionalText(from.getSolution());
to.setAdditionalTextTitle("Solution");
} else if (from.getHint() != null) {
} else if (from.getHint() != null && !from.getHint().isEmpty()) {
to.setAdditionalText(from.getHint());
to.setAdditionalTextTitle("Hint");
}
......@@ -365,18 +395,31 @@ public class FromV2Migrator {
return to;
}
private UserProfile.AuthProvider detectAuthProvider(final String loginId) {
private void updateProfileFromLoginId(final UserProfile profile, final String loginId) {
if (loginId.length() == 15 && loginId.startsWith("Guest")) {
return UserProfile.AuthProvider.ARSNOVA_GUEST;
}
if (loginId.startsWith("https://www.facebook.com/") || loginId.startsWith("http://www.facebook.com/")) {
return UserProfile.AuthProvider.FACEBOOK;
}
if (loginId.contains("@")) {
return UserProfile.AuthProvider.GOOGLE;
profile.setAuthProvider(UserProfile.AuthProvider.ARSNOVA_GUEST);
profile.setLoginId(loginId);
} else if (loginId.startsWith("$")) {
profile.setAuthProvider(UserProfile.AuthProvider.ANONYMIZED);
/* Remove redundant prefix and shorten ID to 10 chars */
profile.setLoginId(loginId.substring(loginId.lastIndexOf("$") + 1).substring(0, 10));
} else if (loginId.startsWith("oidc:")) {
profile.setAuthProvider(UserProfile.AuthProvider.OIDC);
profile.setLoginId(loginId.substring(5));
} else if (loginId.startsWith("saml:")) {
profile.setAuthProvider(UserProfile.AuthProvider.SAML);
profile.setLoginId(loginId.substring(5));
} else if (loginId.startsWith("https://www.facebook.com/") || loginId.startsWith("http://www.facebook.com/")) {
profile.setAuthProvider(UserProfile.AuthProvider.FACEBOOK);
/* Extract ID from URL */
profile.setLoginId(loginId.substring(loginId.indexOf("/", 23) + 1, loginId.length() - 1));
} else if (loginId.contains("@")) {
profile.setAuthProvider(UserProfile.AuthProvider.GOOGLE);
profile.setLoginId(loginId);
} else {
profile.setAuthProvider(UserProfile.AuthProvider.UNKNOWN);
profile.setLoginId(loginId);
}
return UserProfile.AuthProvider.UNKNOWN;
}
public void setIgnoreRevision(final boolean ignoreRevision) {
......
......@@ -19,14 +19,17 @@
package de.thm.arsnova.model.migration;
import static de.thm.arsnova.model.migration.FromV2Migrator.V2_TYPE_ABCD;
import static de.thm.arsnova.model.migration.FromV2Migrator.V2_TYPE_FLASHCARD;
import static de.thm.arsnova.model.migration.FromV2Migrator.V2_TYPE_FREETEXT;
import static de.thm.arsnova.model.migration.FromV2Migrator.V2_TYPE_GRID;
import static de.thm.arsnova.model.migration.FromV2Migrator.V2_TYPE_MC;
import static de.thm.arsnova.model.migration.FromV2Migrator.V2_TYPE_SCHOOL;
import static de.thm.arsnova.model.migration.FromV2Migrator.V2_TYPE_SLIDE;
import static de.thm.arsnova.model.migration.FromV2Migrator.V2_TYPE_VOTE;
import static de.thm.arsnova.model.migration.FromV2Migrator.V2_TYPE_YESNO;
import java.util.ArrayList;
import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
......@@ -210,7 +213,24 @@ public class ToV2Migrator {
to.setQuestionType(V2_TYPE_FREETEXT);
break;
case TEXT:
to.setQuestionType(V2_TYPE_FREETEXT);
final String legacyType = (String) from.getExtensions()
.getOrDefault("v2", Collections.emptyMap()).getOrDefault("format", "");
switch (legacyType) {
case V2_TYPE_SLIDE:
to.setQuestionType(V2_TYPE_SLIDE);
break;
case V2_TYPE_FLASHCARD:
to.setQuestionType(V2_TYPE_FLASHCARD);
final AnswerOption back = new AnswerOption();
back.setText(from.getAdditionalText());
back.setCorrect(true);
to.setPossibleAnswers(Collections.singletonList(back));
break;
default:
to.setQuestionType(V2_TYPE_FREETEXT);
break;
}
break;
default:
throw new IllegalArgumentException("Unsupported content format.");
......
/*
* This file is part of ARSnova Backend.
* Copyright (C) 2012-2019 The ARSnova Team and Contributors
*
* ARSnova Backend is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* ARSnova Backend is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package de.thm.arsnova.persistence.couchdb.migrations;
import org.springframework.context.annotation.Condition;
import org.springframework.context.annotation.ConditionContext;
import org.springframework.core.type.AnnotatedTypeMetadata;
import org.springframework.stereotype.Component;
@Component
public class MigrateFromLegacyCondition implements Condition {
@Override
public boolean matches(final ConditionContext context, final AnnotatedTypeMetadata metadata) {
final String migrateFrom = context.getEnvironment().getProperty("couchdb.migrate-from");
return migrateFrom != null && !migrateFrom.isEmpty();
}
}
......@@ -19,6 +19,7 @@
package de.thm.arsnova.persistence.couchdb.migrations;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
......@@ -31,12 +32,14 @@ import org.ektorp.DocumentNotFoundException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.context.annotation.Conditional;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.stereotype.Service;
import de.thm.arsnova.config.properties.CouchDbProperties;
import de.thm.arsnova.model.Answer;
import de.thm.arsnova.model.Comment;
import de.thm.arsnova.model.Content;
import de.thm.arsnova.model.ContentGroup;
import de.thm.arsnova.model.Motd;
import de.thm.arsnova.model.Room;
import de.thm.arsnova.model.UserProfile;
......@@ -55,8 +58,10 @@ import de.thm.arsnova.persistence.couchdb.support.PagedMangoResponse;
*
* @author Daniel Gerhardt
*/
@Conditional(MigrateFromLegacyCondition.class)
@Service
@ConditionalOnProperty(
name = "migrate-from",
prefix = CouchDbProperties.PREFIX)
public class V2ToV3Migration implements Migration {
private static final String ID = "20170914131300";
private static final int LIMIT = 200;
......@@ -65,6 +70,7 @@ public class V2ToV3Migration implements Migration {
private static final String USER_INDEX = "user-index";
private static final String LOGGEDIN_INDEX = "loggedin-index";
private static final String SESSION_INDEX = "session-index";
private static final String SKILLQUESTION_INDEX = "skillquestion-index";
private static final String MOTD_INDEX = "motd-index";
private static final String MOTDLIST_INDEX = "motdlist-index";
......@@ -107,6 +113,7 @@ public class V2ToV3Migration implements Migration {
migrateMotds();
migrateComments();
migrateContents();
migrateContentGroups();
migrateAnswers();
} catch (final InterruptedException e) {
throw new DbAccessException(e);
......@@ -148,6 +155,14 @@ public class V2ToV3Migration implements Migration {
fields.add(new MangoCouchDbConnector.MangoQuery.Sort("keyword", false));
fromConnector.createPartialJsonIndex(SESSION_INDEX, fields, filterSelector);
filterSelector.clear();
filterSelector.put("type", "skill_question");
fromConnector.createPartialJsonIndex(SKILLQUESTION_INDEX, new ArrayList<>(), filterSelector);
fields.clear();
fields.add(new MangoCouchDbConnector.MangoQuery.Sort("sessionId", false));
fields.add(new MangoCouchDbConnector.MangoQuery.Sort("questionVariant", false));
fromConnector.createPartialJsonIndex(SKILLQUESTION_INDEX, fields, filterSelector);
filterSelector.clear();
filterSelector.put("type", "motd");
fromConnector.createPartialJsonIndex(MOTD_INDEX, new ArrayList<>(), filterSelector);
......@@ -428,6 +443,64 @@ public class V2ToV3Migration implements Migration {
}
}
private void migrateContentGroups() throws InterruptedException {
waitForV2Index(SKILLQUESTION_INDEX);
final Map<String, Object> queryOptions = new HashMap<>();
queryOptions.put("type", "skill_question");
final MangoCouchDbConnector.MangoQuery query = new MangoCouchDbConnector.MangoQuery(queryOptions);
query.setFields(Arrays.asList("sessionId", "questionVariant", "_id"));
final ArrayList<MangoCouchDbConnector.MangoQuery.Sort> sort = new ArrayList<>();
sort.add(new MangoCouchDbConnector.MangoQuery.Sort("sessionId", false));
sort.add(new MangoCouchDbConnector.MangoQuery.Sort("questionVariant", false));
query.setSort(sort);
query.setIndexDocument(SKILLQUESTION_INDEX);
query.setLimit(LIMIT);
String bookmark = null;
final Map<String, Set<String>> groups = new HashMap<>();
String roomId = "";
for (int skip = 0;; skip += LIMIT) {
logger.debug("Migration progress: {}, bookmark: {}", skip, bookmark);
query.setBookmark(bookmark);
final PagedMangoResponse<de.thm.arsnova.model.migration.v2.Content> response =
fromConnector.queryForPage(query, de.thm.arsnova.model.migration.v2.Content.class);
final List<de.thm.arsnova.model.migration.v2.Content> contentsV2 = response.getEntities();
bookmark = response.getBookmark();
if (contentsV2.size() == 0) {
break;
}
for (final de.thm.arsnova.model.migration.v2.Content contentV2 : contentsV2) {
if (!contentRepository.existsById(contentV2.getId())) {
continue;
}
if (!contentV2.getSession().equals(roomId)) {
createContentGroups(roomId, groups);
roomId = contentV2.getSessionId();
}
final Set<String> contentIds = groups.getOrDefault(contentV2.getQuestionVariant(), new HashSet<>());
groups.put(contentV2.getQuestionVariant(), contentIds);
contentIds.add(contentV2.getId());
}
}
createContentGroups(roomId, groups);
}
private void createContentGroups(final String roomId, final Map<String, Set<String>> groups) {
if (!groups.isEmpty()) {
final List<ContentGroup> contentGroups = new ArrayList<>();
for (final String name : groups.keySet()) {
final ContentGroup group = new ContentGroup();
group.setRoomId(roomId);
group.setName(name);
group.setAutoSort(true);
group.setContentIds(groups.get(name));
contentGroups.add(group);
}
toConnector.executeBulk(contentGroups);
groups.clear();
}
}
private void migrateAnswers() throws InterruptedException {
waitForV2Index(FULL_INDEX_BY_TYPE);
final Map<String, Object> queryOptions = new HashMap<>();
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment