Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

add final team snapshots #178

Merged
merged 4 commits into from
Dec 11, 2024
Merged

add final team snapshots #178

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
4 commits
Select commit Hold shift click to select a range
3d8c8f0
move history length manipulation from xAll to xEach
nephest Dec 10, 2024
19995b8
archive last snapshots
nephest Dec 10, 2024
5fa093e
create last season snapshots
nephest Dec 10, 2024
84b1690
use team_state instead of derived teams
nephest Dec 11, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
create last season snapshots
  • Loading branch information
@nephest
nephest committed Dec 11, 2024
commit 5fa093e302384b0c7d8db955a9b10893b78b6a57
107 changes: 95 additions & 12 deletions src/main/java/com/nephest/battlenet/sc2/web/service/TeamStateService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 21,10 @@
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.function.BiConsumer;
import java.util.function.Function;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
Expand All @@ -40,6 43,9 @@ public class TeamStateService
private static final Logger LOG = LoggerFactory.getLogger(TeamStateService.class);

public static final int TEAM_ARCHIVE_BATCH_SIZE = 500;
public static final int FINAL_TEAM_STATE_BATCH_SIZE = TEAM_ARCHIVE_BATCH_SIZE;
public static final Duration FINAL_TEAM_SNAPSHOT_OFFSET
= TeamDAO.MIN_DURATION_BETWEEN_SEASONS.dividedBy(2);

private final SeasonDAO seasonDAO;
private final TeamDAO teamDAO;
Expand All @@ -49,6 55,7 @@ public class TeamStateService
private TeamStateService service;
private int mainLengthDays, secondaryLengthDays;

private final Map<Region, LongVar> lastFinalizedSeason = new EnumMap<>(Region.class);
private final Map<Region, LongVar> lastArchiveSeason = new EnumMap<>(Region.class);
private InstantVar lastClearInstant;
private final Sinks.Many<LadderUpdateData> updateEvent = Sinks
Expand Down Expand Up @@ -85,6 92,16 @@ private void initVars(VarDAO varDAO)
{
for(Region region : Region.values())
{
lastFinalizedSeason.put
(
region,
new LongVar
(
varDAO,
region.getId() ".mmr.history.finalized.season",
false
)
);
lastArchiveSeason.put
(
region,
Expand All @@ -103,9 120,12 @@ private void initVars(VarDAO varDAO)
false
);

lastArchiveSeason.values().forEach(var->{
var.tryLoad();
if(var.getValue() == null) var.setValue(0L);
Stream.of(lastFinalizedSeason, lastArchiveSeason)
.map(Map::values)
.flatMap(Collection::stream)
.forEach(var->{
var.tryLoad();
if(var.getValue() == null) var.setValue(0L);
});

lastClearInstant.tryLoad();
Expand Down Expand Up @@ -133,10 153,23 @@ protected void setService(TeamStateService service)

protected void reset()
{
lastArchiveSeason.values().forEach(v->v.setValueAndSave(Long.MIN_VALUE));
Stream.of(lastFinalizedSeason, lastArchiveSeason)
.map(Map::values)
.flatMap(Collection::stream)
.forEach(v->v.setValueAndSave(Long.MIN_VALUE));
lastClearInstant.setValueAndSave(Instant.MIN);
}

protected Map<Region, LongVar> getLastFinalizedSeasonVars()
{
return lastFinalizedSeason;
}

protected Map<Region, LongVar> getLastArchiveSeasonVars()
{
return lastArchiveSeason;
}

protected InstantVar getLastClearInstantVar()
{
return lastClearInstant;
Expand Down Expand Up @@ -168,12 201,6 @@ public Flux<LadderUpdateData> getUpdateEvent()
}

private void update(LadderUpdateData data)
{
updateArchive(data);
removeExpired();
}

private void updateArchive(LadderUpdateData data)
{
Map<Region, Set<Integer>> updates = data.getContexts().stream()
.map(Map::entrySet)
Expand All @@ -183,20 210,76 @@ private void updateArchive(LadderUpdateData data)
()->new EnumMap<>(Region.class),
Collectors.mapping(entry->entry.getValue().getSeason().getBattlenetId(), Collectors.toSet())
));
takeFinalTeamSnapshots(updates);
updateArchive(updates);
removeExpired();
}

private void processUpdates
(
Map<Region, Set<Integer>> updates,
Function<Region, Integer> minSeasonFunction,
BiConsumer<Region, Integer> updateTask
)
{
updates.forEach((region, seasons) -> {
//next after previous archive
long minSeason = this.lastArchiveSeason.get(region).getValue() 1;
long minSeason = minSeasonFunction.apply(region);
//not current season
int maxSeason = seasonDAO.getMaxBattlenetId(region) - 1;
seasons.stream()
//update only previous seasons that have ended already
.map(season->season - 1)
.filter(season->season >= minSeason && season <= maxSeason)
.sorted()
.forEach(season->service.updateArchive(region, season));
.forEach(season->updateTask.accept(region, season));
});
}

private void takeFinalTeamSnapshots(Map<Region, Set<Integer>> updates)
{
processUpdates
(
updates,
r->lastFinalizedSeason.get(r).getValue().intValue() 1,
(region, season)->service.takeFinalTeamSnapshots(region, season)
);
}

@Transactional
public void takeFinalTeamSnapshots(Region region, int season)
{
List<Long> teamIds = teamDAO.findIds(region, season);
if(teamIds.isEmpty())
{
lastFinalizedSeason.get(region).setValueAndSave((long) season);
return;
}

OffsetDateTime odt = teamDAO.findMaxLastPlayed(region, season)
.orElseThrow()
.plus(FINAL_TEAM_SNAPSHOT_OFFSET);
for(int i = 0; i < teamIds.size(); )
{
LOG.trace("Final team states {} {} progress: {}/{}", region, season, i, teamIds.size());
int nextIx = Math.min((i 1) * FINAL_TEAM_STATE_BATCH_SIZE, teamIds.size());
teamStateDAO.takeSnapshot(teamIds.subList(i, nextIx), odt);
i = nextIx;
}
lastFinalizedSeason.get(region).setValueAndSave((long) season);
LOG.info("Created final team states: {} {}", region, season);
}

private void updateArchive(Map<Region, Set<Integer>> updates)
{
processUpdates
(
updates,
r->lastArchiveSeason.get(r).getValue().intValue() 1,
(region, season)->service.updateArchive(region, season)
);
}

@Transactional
public void updateArchive(Region region, int season)
{
Expand Down
72 changes: 72 additions & 0 deletions src/main/sql/1.61.0-1.62.0-migration.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -146,3 146,75 @@ END LOOP;
END
$do$
LANGUAGE plpgsql;

DO
$do$
DECLARE
seasonId INTEGER;
createdCurrent INTEGER;
BEGIN
FOR seasonId IN SELECT DISTINCT(battlenet_id) FROM season ORDER BY battlenet_id LOOP

createdCurrent := 0;
WITH snapshot_timestamp AS
(
SELECT team.region,
COALESCE(MAX(timestamp), MAX(last_played), MAX(season."end")) INTERVAL '1 second' AS timestamp
FROM team
LEFT JOIN team_state ON team.id = team_state.team_id
INNER JOIN season ON team.region = season.region AND team.season = season.battlenet_id
WHERE season = seasonId
GROUP BY team.region
),
current_season AS
(
SELECT region, MAX(battlenet_id) AS battlenet_id
FROM season
GROUP BY region
),
last_snapshots AS
(
INSERT INTO team_state
(
"team_id",
"timestamp",
"division_id",
"population_state_id",
"wins",
"games",
"rating",
"global_rank",
"region_rank",
"league_rank",
"secondary"
)
SELECT
"id",
"timestamp",
"division_id",
"population_state_id",
"wins",
"wins" "losses" "ties",
"rating",
"global_rank",
"region_rank",
"league_rank",
CASE WHEN queue_type != 201 THEN true ELSE NULL END
FROM team
INNER JOIN snapshot_timestamp USING(region)
INNER JOIN current_season USING(region)
WHERE season = seasonId
AND season != current_season.battlenet_id
RETURNING team_id, timestamp
)
INSERT INTO team_state_archive(team_id, timestamp)
SELECT team_id, timestamp
FROM last_snapshots;

GET DIAGNOSTICS createdCurrent = ROW_COUNT;
RAISE NOTICE 'Updated season %, created %', seasonId, createdCurrent;
END LOOP;

END
$do$
LANGUAGE plpgsql;
4 changes: 4 additions & 0 deletions src/test/java/com/nephest/battlenet/sc2/config/SpyBeanConfig.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 5,7 @@

import com.nephest.battlenet.sc2.model.local.dao.SeasonDAO;
import com.nephest.battlenet.sc2.model.local.dao.TeamStateArchiveDAO;
import com.nephest.battlenet.sc2.model.local.dao.TeamStateDAO;
import com.nephest.battlenet.sc2.model.local.dao.VarDAO;
import com.nephest.battlenet.sc2.web.service.BlizzardSC2API;
import com.nephest.battlenet.sc2.web.service.SC2ArcadeAPI;
Expand All @@ -25,6 26,9 @@ public class SpyBeanConfig
@SpyBean
private VarDAO varDAO;

@SpyBean
private TeamStateDAO teamStateDAO;

@SpyBean
private TeamStateArchiveDAO teamStateArchiveDAO;

Expand Down
Loading