Commit 77136ccf authored by Daniel Gerhardt's avatar Daniel Gerhardt

Merge branch 'migration-state' into 'master'

Continue aborted migrations with help of saved state

See merge request arsnova/arsnova-backend!179
parents 0a4c5e47 ddbf9133
......@@ -23,6 +23,7 @@ import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.Objects;
import org.springframework.core.style.ToStringCreator;
import de.thm.arsnova.model.serialization.View;
......@@ -30,6 +31,8 @@ public class MigrationState extends Entity {
public static class Migration {
private String id;
private Date start;
private int step;
private Object state;
public Migration() {
......@@ -50,9 +53,34 @@ public class MigrationState extends Entity {
return start;
}
@JsonView({View.Persistence.class, View.Public.class})
public int getStep() {
return step;
}
@JsonView(View.Persistence.class)
public void setStep(final int step) {
this.step = step;
}
@JsonView({View.Persistence.class, View.Public.class})
public Object getState() {
return state;
}
@JsonView(View.Persistence.class)
public void setState(final Object state) {
this.state = state;
}
@Override
public String toString() {
return "Migration " + id + " started at " + start;
return new ToStringCreator(this)
.append("id", id)
.append("start", start)
.append("step", step)
.append("state", state)
.toString();
}
}
......@@ -134,4 +162,11 @@ public class MigrationState extends Entity {
return Objects.equals(active, that.active)
&& Objects.equals(completed, that.completed);
}
@Override
protected ToStringCreator buildToString() {
return super.buildToString()
.append("active", active)
.append("completed", completed);
}
}
......@@ -132,8 +132,8 @@ public class CouchDbInitializer implements ResourceLoaderAware {
}
protected void migrate(final MigrationState state) {
if (migrationExecutor != null && migrationExecutor.runMigrations(state)) {
connector.update(state);
if (migrationExecutor != null) {
migrationExecutor.runMigrations(state, () -> connector.update(state));
}
}
......@@ -158,8 +158,8 @@ public class CouchDbInitializer implements ResourceLoaderAware {
migrate(state);
statusService.removeMaintenanceReason(this.getClass());
} catch (final DbAccessException e) {
logger.error("Database is invalid.", e);
statusService.putMaintenanceReason(this.getClass(), "Invalid database");
logger.error("Database initialization failed.", e);
statusService.putMaintenanceReason(this.getClass(), "Invalid database state");
}
}
......
......@@ -18,8 +18,12 @@
package de.thm.arsnova.persistence.couchdb.migrations;
import de.thm.arsnova.model.MigrationState;
public interface Migration {
String getId();
void migrate();
int getStepCount();
void migrate(MigrationState.Migration state);
}
......@@ -48,22 +48,43 @@ public class MigrationExecutor {
logger.debug("Initialized {} migration(s).", this.migrations.size());
}
public boolean runMigrations(@NonNull final MigrationState migrationState) {
public boolean runMigrations(@NonNull final MigrationState migrationState, final Runnable stateUpdateHandler) {
final Thread shutdownHook = new Thread(stateUpdateHandler);
Runtime.getRuntime().addShutdownHook(shutdownHook);
final List<Migration> pendingMigrations = migrations.stream()
.filter(m -> !migrationState.getCompleted().contains(m.getId())).collect(Collectors.toList());
boolean stateChange = false;
if (migrationState.getActive() != null) {
throw new IllegalStateException("An migration is already active: " + migrationState.getActive());
}
logger.debug("Pending migrations: " + pendingMigrations.stream()
logger.info("Pending migrations: " + pendingMigrations.stream()
.map(Migration::getId).collect(Collectors.joining()));
for (final Migration migration : pendingMigrations) {
if (migrationState.getActive() != null) {
logger.info("Trying to continue from aborted migration: " + migrationState.getActive());
if (pendingMigrations.isEmpty()
|| migrationState.getActive().getId().equals(pendingMigrations.get(0))) {
throw new IllegalStateException("Migration state does not match next pending migration.");
}
} else {
migrationState.setActive(migration.getId(), new Date());
}
stateChange = true;
migrationState.setActive(migration.getId(), new Date());
migration.migrate();
final int initialStep = migrationState.getActive() != null ? migrationState.getActive().getStep() : 0;
for (int i = initialStep; i < migration.getStepCount(); i++) {
logger.info("Performing migration {} step {}...", migration.getId(), i);
try {
migration.migrate(migrationState.getActive());
} catch (final Exception e) {
logger.info("Current migration state: {}", migrationState);
stateUpdateHandler.run();
throw e;
}
migrationState.getActive().setStep(i + 1);
logger.info("Completed migration {} step {}.", migration.getId(), i);
stateUpdateHandler.run();
}
migrationState.getCompleted().add(migration.getId());
migrationState.setActive(null);
}
Runtime.getRuntime().removeShutdownHook(shutdownHook);
return stateChange;
}
......
......@@ -40,6 +40,7 @@ import de.thm.arsnova.model.Answer;
import de.thm.arsnova.model.Comment;
import de.thm.arsnova.model.Content;
import de.thm.arsnova.model.ContentGroup;
import de.thm.arsnova.model.MigrationState;
import de.thm.arsnova.model.Motd;
import de.thm.arsnova.model.Room;
import de.thm.arsnova.model.UserProfile;
......@@ -83,6 +84,7 @@ public class V2ToV3Migration implements Migration {
private RoomRepository roomRepository;
private ContentRepository contentRepository;
private long referenceTimestamp = System.currentTimeMillis();
private MigrationState.Migration state;
public V2ToV3Migration(
final FromV2Migrator migrator,
......@@ -103,18 +105,44 @@ public class V2ToV3Migration implements Migration {
return ID;
}
public void migrate() {
public int getStepCount() {
return 8;
}
@Override
public void migrate(final MigrationState.Migration state) {
this.state = state;
createV2Index();
migrator.setIgnoreRevision(true);
try {
migrateUsers();
migrateUnregisteredUsers();
migrateRooms();
migrateMotds();
migrateComments();
migrateContents();
migrateContentGroups();
migrateAnswers();
switch (state.getStep()) {
case 0:
migrateUsers();
break;
case 1:
migrateUnregisteredUsers();
break;
case 2:
migrateRooms();
break;
case 3:
migrateMotds();
break;
case 4:
migrateComments();
break;
case 5:
migrateContents();
break;
case 6:
migrateContentGroups();
break;
case 7:
migrateAnswers();
break;
default:
throw new IllegalStateException("Invalid migration step:" + state.getStep() + ".");
}
} catch (final InterruptedException e) {
throw new DbAccessException(e);
}
......@@ -195,7 +223,7 @@ public class V2ToV3Migration implements Migration {
final MangoCouchDbConnector.MangoQuery query = new MangoCouchDbConnector.MangoQuery(queryOptions);
query.setIndexDocument(USER_INDEX);
query.setLimit(LIMIT);
String bookmark = null;
String bookmark = (String) state.getState();
for (int skip = 0;; skip += LIMIT) {
logger.debug("Migration progress: {}, bookmark: {}", skip, bookmark);
......@@ -225,7 +253,9 @@ public class V2ToV3Migration implements Migration {
}
toConnector.executeBulk(profilesV3);
state.setState(bookmark);
}
state.setState(null);
}
private void migrateUnregisteredUsers() throws InterruptedException {
......@@ -253,7 +283,7 @@ public class V2ToV3Migration implements Migration {
query = new MangoCouchDbConnector.MangoQuery(queryOptions);
query.setIndexDocument(LOGGEDIN_INDEX);
query.setLimit(LIMIT);
String bookmark = null;
String bookmark = (String) state.getState();
for (int skip = 0;; skip += LIMIT) {
logger.debug("Migration progress: {}, bookmark: {}", skip, bookmark);
query.setBookmark(bookmark);
......@@ -276,7 +306,9 @@ public class V2ToV3Migration implements Migration {
profilesV3.add(profileV3);
}
toConnector.executeBulk(profilesV3);
state.setState(bookmark);
}
state.setState(null);
}
private void migrateRooms() throws InterruptedException {
......@@ -286,7 +318,7 @@ public class V2ToV3Migration implements Migration {
final MangoCouchDbConnector.MangoQuery query = new MangoCouchDbConnector.MangoQuery(queryOptions);
query.setIndexDocument(SESSION_INDEX);
query.setLimit(LIMIT);
String bookmark = null;
String bookmark = (String) state.getState();
for (int skip = 0;; skip += LIMIT) {
logger.debug("Migration progress: {}, bookmark: {}", skip, bookmark);
......@@ -311,7 +343,9 @@ public class V2ToV3Migration implements Migration {
}
toConnector.executeBulk(roomsV3);
state.setState(bookmark);
}
state.setState(null);
}
private void migrateMotds() throws InterruptedException {
......@@ -325,7 +359,7 @@ public class V2ToV3Migration implements Migration {
final MangoCouchDbConnector.MangoQuery query = new MangoCouchDbConnector.MangoQuery(queryOptions);
query.setIndexDocument(MOTD_INDEX);
query.setLimit(LIMIT);
String bookmark = null;
String bookmark = (String) state.getState();
for (int skip = 0;; skip += LIMIT) {
logger.debug("Migration progress: {}, bookmark: {}", skip, bookmark);
......@@ -354,7 +388,9 @@ public class V2ToV3Migration implements Migration {
}
toConnector.executeBulk(motdsV3);
state.setState(bookmark);
}
state.setState(null);
}
private void migrateComments() throws InterruptedException {
......@@ -364,7 +400,7 @@ public class V2ToV3Migration implements Migration {
final MangoCouchDbConnector.MangoQuery query = new MangoCouchDbConnector.MangoQuery(queryOptions);
query.setIndexDocument(FULL_INDEX_BY_TYPE);
query.setLimit(LIMIT);
String bookmark = null;
String bookmark = (String) state.getState();
for (int skip = 0;; skip += LIMIT) {
logger.debug("Migration progress: {}, bookmark: {}", skip, bookmark);
......@@ -402,7 +438,9 @@ public class V2ToV3Migration implements Migration {
}
toConnector.executeBulk(commentsV3);
state.setState(bookmark);
}
state.setState(null);
}
private void migrateContents() throws InterruptedException {
......@@ -412,7 +450,7 @@ public class V2ToV3Migration implements Migration {
final MangoCouchDbConnector.MangoQuery query = new MangoCouchDbConnector.MangoQuery(queryOptions);
query.setIndexDocument(FULL_INDEX_BY_TYPE);
query.setLimit(LIMIT);
String bookmark = null;
String bookmark = (String) state.getState();
for (int skip = 0;; skip += LIMIT) {
logger.debug("Migration progress: {}, bookmark: {}", skip, bookmark);
......@@ -440,7 +478,9 @@ public class V2ToV3Migration implements Migration {
}
toConnector.executeBulk(contentsV3);
state.setState(bookmark);
}
state.setState(null);
}
private void migrateContentGroups() throws InterruptedException {
......@@ -455,7 +495,7 @@ public class V2ToV3Migration implements Migration {
query.setSort(sort);
query.setIndexDocument(SKILLQUESTION_INDEX);
query.setLimit(LIMIT);
String bookmark = null;
String bookmark = (String) state.getState();
final Map<String, Set<String>> groups = new HashMap<>();
String roomId = "";
......@@ -481,8 +521,10 @@ public class V2ToV3Migration implements Migration {
groups.put(contentV2.getQuestionVariant(), contentIds);
contentIds.add(contentV2.getId());
}
state.setState(bookmark);
}
createContentGroups(roomId, groups);
state.setState(null);
}
private void createContentGroups(final String roomId, final Map<String, Set<String>> groups) {
......@@ -508,7 +550,7 @@ public class V2ToV3Migration implements Migration {
final MangoCouchDbConnector.MangoQuery query = new MangoCouchDbConnector.MangoQuery(queryOptions);
query.setIndexDocument(FULL_INDEX_BY_TYPE);
query.setLimit(LIMIT);
String bookmark = null;
String bookmark = (String) state.getState();
for (int skip = 0;; skip += LIMIT) {
logger.debug("Migration progress: {}, bookmark: {}", skip, bookmark);
......@@ -541,7 +583,9 @@ public class V2ToV3Migration implements Migration {
}
toConnector.executeBulk(answersV3);
state.setState(bookmark);
}
state.setState(null);
}
private HashSet<String> migrateMotdIds(final Set<String> oldIds) throws InterruptedException {
......
......@@ -20,6 +20,7 @@ package de.thm.arsnova.persistence.couchdb.support;
import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.core.JsonToken;
import com.fasterxml.jackson.databind.JsonMappingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import java.io.IOException;
import java.io.InputStream;
......@@ -77,8 +78,13 @@ public class MangoQueryResultParser<T> {
while (jp.nextValue() != JsonToken.END_OBJECT) {
final String currentName = jp.getCurrentName();
if (DOCS_FIELD_NAME.equals(currentName)) {
docs = new ArrayList<T>();
parseDocs(jp);
docs = new ArrayList<>();
try {
parseDocs(jp);
} catch (final JsonMappingException e) {
logger.error("Failed to map document at index {}.", docs.size());
throw e;
}
} else if (BOOKMARK_FIELD_NAME.equals(currentName)) {
bookmark = jp.getText();
} else if (WARNING_FIELD_NAME.equals(currentName)) {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment