Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: db migration #3595

Merged
merged 34 commits into from
Jul 6, 2024
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
34 commits
Select commit Hold shift click to select a range
df09d08
feat(sqlalchemy): Replace peewee with sqlalchemy
jonathan-rohde Jun 18, 2024
bee835c
feat(sqlalchemy): remove session reference from router
jonathan-rohde Jun 21, 2024
070d908
feat(sqlalchemy): use subprocess to do migrations
jonathan-rohde Jun 24, 2024
320e658
feat(sqlalchemy): cleanup fixes
jonathan-rohde Jun 24, 2024
c134eab
feat(sqlalchemy): format backend
jonathan-rohde Jun 24, 2024
eb01e8d
feat(sqlalchemy): use scoped session
jonathan-rohde Jun 24, 2024
da403f3
feat(sqlalchemy): use session factory instead of context manager
jonathan-rohde Jun 24, 2024
a9b1487
feat(sqlalchemy): fix wrong column types
jonathan-rohde Jun 24, 2024
8f939cf
feat(sqlalchemy): some fixes
jonathan-rohde Jun 24, 2024
2fb27ad
feat(sqlalchemy): add missing file
jonathan-rohde Jun 24, 2024
d88bd51
feat(sqlalchemy): format backend
jonathan-rohde Jun 24, 2024
642c352
feat(sqlalchemy): rebase
jonathan-rohde Jun 25, 2024
d4b6b7c
feat(sqlalchemy): reverted not needed api change
jonathan-rohde Jun 25, 2024
23e4d9d
feat(sqlalchemy): formatting
jonathan-rohde Jun 25, 2024
827b1e5
feat(sqlalchemy): execute tests in github actions
jonathan-rohde Jun 25, 2024
df47c49
Merge branch 'refs/heads/dev' into feat/sqlalchemy-instead-of-peewee
jonathan-rohde Jun 28, 2024
5391f4c
feat(sqlalchemy): add new column
jonathan-rohde Jun 28, 2024
2aecd7d
Merge branch 'refs/heads/dev' into feat/sqlalchemy-instead-of-peewee
jonathan-rohde Jul 1, 2024
d0e89a0
Merge pull request #3327 from jonathan-rohde/feat/sqlalchemy-instead-…
tjbck Jul 2, 2024
647aa19
chore: format
tjbck Jul 2, 2024
44a9b86
fix: functions
tjbck Jul 3, 2024
aa88022
fix: functions
tjbck Jul 3, 2024
4d23957
revert: model_validate
tjbck Jul 3, 2024
0d78b63
Merge pull request #3621 from open-webui/dev
tjbck Jul 4, 2024
15f6f7b
revert: peewee migrations
tjbck Jul 4, 2024
bfc53b4
revert
tjbck Jul 4, 2024
1b65df3
revert
tjbck Jul 4, 2024
8646460
refac
tjbck Jul 4, 2024
37a5d2c
Update db.py
tjbck Jul 4, 2024
8fe2a7b
fix
tjbck Jul 4, 2024
8b13755
Update auths.py
tjbck Jul 4, 2024
d60f066
Merge pull request #3668 from open-webui/dev
tjbck Jul 6, 2024
1436bb7
enh: handle peewee migration
tjbck Jul 6, 2024
4e75150
Merge pull request #3669 from open-webui/dev-migration-session
tjbck Jul 6, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
revert
  • Loading branch information
tjbck committed Jul 4, 2024
commit bfc53b49fd05e033537415db020972f89be1bd22
21 changes: 3 additions & 18 deletions backend/apps/webui/internal/migrations/004_add_archived.py
Original file line number Diff line number Diff line change
@@ -1,4 1,4 @@
"""Peewee migrations -- 009_add_models.py.
"""Peewee migrations -- 002_add_local_sharing.py.

Some examples (model - class or model name)::

Expand Down Expand Up @@ -37,25 37,10 @@
def migrate(migrator: Migrator, database: pw.Database, *, fake=False):
"""Write your migrations here."""

@migrator.create_model
class Model(pw.Model):
id = pw.TextField(unique=True)
user_id = pw.TextField()
base_model_id = pw.TextField(null=True)

name = pw.TextField()

meta = pw.TextField()
params = pw.TextField()

created_at = pw.BigIntegerField(null=False)
updated_at = pw.BigIntegerField(null=False)

class Meta:
table_name = "model"
migrator.add_fields("chat", archived=pw.BooleanField(default=False))


def rollback(migrator: Migrator, database: pw.Database, *, fake=False):
"""Write your rollback migrations here."""

migrator.remove_model("model")
migrator.remove_fields("chat", "archived")
97 changes: 83 additions & 14 deletions backend/apps/webui/internal/migrations/005_add_updated_at.py
Original file line number Diff line number Diff line change
@@ -1,4 1,4 @@
"""Peewee migrations -- 009_add_models.py.
"""Peewee migrations -- 002_add_local_sharing.py.

Some examples (model - class or model name)::

Expand Down Expand Up @@ -37,25 37,94 @@
def migrate(migrator: Migrator, database: pw.Database, *, fake=False):
"""Write your migrations here."""

@migrator.create_model
class Model(pw.Model):
id = pw.TextField(unique=True)
user_id = pw.TextField()
base_model_id = pw.TextField(null=True)
if isinstance(database, pw.SqliteDatabase):
migrate_sqlite(migrator, database, fake=fake)
else:
migrate_external(migrator, database, fake=fake)

name = pw.TextField()

meta = pw.TextField()
params = pw.TextField()
def migrate_sqlite(migrator: Migrator, database: pw.Database, *, fake=False):
# Adding fields created_at and updated_at to the 'chat' table
migrator.add_fields(
"chat",
created_at=pw.DateTimeField(null=True), # Allow null for transition
updated_at=pw.DateTimeField(null=True), # Allow null for transition
)

created_at = pw.BigIntegerField(null=False)
updated_at = pw.BigIntegerField(null=False)
# Populate the new fields from an existing 'timestamp' field
migrator.sql(
"UPDATE chat SET created_at = timestamp, updated_at = timestamp WHERE timestamp IS NOT NULL"
)

class Meta:
table_name = "model"
# Now that the data has been copied, remove the original 'timestamp' field
migrator.remove_fields("chat", "timestamp")

# Update the fields to be not null now that they are populated
migrator.change_fields(
"chat",
created_at=pw.DateTimeField(null=False),
updated_at=pw.DateTimeField(null=False),
)


def migrate_external(migrator: Migrator, database: pw.Database, *, fake=False):
# Adding fields created_at and updated_at to the 'chat' table
migrator.add_fields(
"chat",
created_at=pw.BigIntegerField(null=True), # Allow null for transition
updated_at=pw.BigIntegerField(null=True), # Allow null for transition
)

# Populate the new fields from an existing 'timestamp' field
migrator.sql(
"UPDATE chat SET created_at = timestamp, updated_at = timestamp WHERE timestamp IS NOT NULL"
)

# Now that the data has been copied, remove the original 'timestamp' field
migrator.remove_fields("chat", "timestamp")

# Update the fields to be not null now that they are populated
migrator.change_fields(
"chat",
created_at=pw.BigIntegerField(null=False),
updated_at=pw.BigIntegerField(null=False),
)


def rollback(migrator: Migrator, database: pw.Database, *, fake=False):
"""Write your rollback migrations here."""

migrator.remove_model("model")
if isinstance(database, pw.SqliteDatabase):
rollback_sqlite(migrator, database, fake=fake)
else:
rollback_external(migrator, database, fake=fake)


def rollback_sqlite(migrator: Migrator, database: pw.Database, *, fake=False):
# Recreate the timestamp field initially allowing null values for safe transition
migrator.add_fields("chat", timestamp=pw.DateTimeField(null=True))

# Copy the earliest created_at date back into the new timestamp field
# This assumes created_at was originally a copy of timestamp
migrator.sql("UPDATE chat SET timestamp = created_at")

# Remove the created_at and updated_at fields
migrator.remove_fields("chat", "created_at", "updated_at")

# Finally, alter the timestamp field to not allow nulls if that was the original setting
migrator.change_fields("chat", timestamp=pw.DateTimeField(null=False))


def rollback_external(migrator: Migrator, database: pw.Database, *, fake=False):
# Recreate the timestamp field initially allowing null values for safe transition
migrator.add_fields("chat", timestamp=pw.BigIntegerField(null=True))

# Copy the earliest created_at date back into the new timestamp field
# This assumes created_at was originally a copy of timestamp
migrator.sql("UPDATE chat SET timestamp = created_at")

# Remove the created_at and updated_at fields
migrator.remove_fields("chat", "created_at", "updated_at")

# Finally, alter the timestamp field to not allow nulls if that was the original setting
migrator.change_fields("chat", timestamp=pw.BigIntegerField(null=False))
Original file line number Diff line number Diff line change
@@ -1,4 1,4 @@
"""Peewee migrations -- 009_add_models.py.
"""Peewee migrations -- 006_migrate_timestamps_and_charfields.py.

Some examples (model - class or model name)::

Expand Down Expand Up @@ -37,25 37,94 @@
def migrate(migrator: Migrator, database: pw.Database, *, fake=False):
"""Write your migrations here."""

@migrator.create_model
class Model(pw.Model):
id = pw.TextField(unique=True)
user_id = pw.TextField()
base_model_id = pw.TextField(null=True)

name = pw.TextField()

meta = pw.TextField()
params = pw.TextField()

created_at = pw.BigIntegerField(null=False)
updated_at = pw.BigIntegerField(null=False)

class Meta:
table_name = "model"
# Alter the tables with timestamps
migrator.change_fields(
"chatidtag",
timestamp=pw.BigIntegerField(),
)
migrator.change_fields(
"document",
timestamp=pw.BigIntegerField(),
)
migrator.change_fields(
"modelfile",
timestamp=pw.BigIntegerField(),
)
migrator.change_fields(
"prompt",
timestamp=pw.BigIntegerField(),
)
migrator.change_fields(
"user",
timestamp=pw.BigIntegerField(),
)
# Alter the tables with varchar to text where necessary
migrator.change_fields(
"auth",
password=pw.TextField(),
)
migrator.change_fields(
"chat",
title=pw.TextField(),
)
migrator.change_fields(
"document",
title=pw.TextField(),
filename=pw.TextField(),
)
migrator.change_fields(
"prompt",
title=pw.TextField(),
)
migrator.change_fields(
"user",
profile_image_url=pw.TextField(),
)


def rollback(migrator: Migrator, database: pw.Database, *, fake=False):
"""Write your rollback migrations here."""

migrator.remove_model("model")
if isinstance(database, pw.SqliteDatabase):
# Alter the tables with timestamps
migrator.change_fields(
"chatidtag",
timestamp=pw.DateField(),
)
migrator.change_fields(
"document",
timestamp=pw.DateField(),
)
migrator.change_fields(
"modelfile",
timestamp=pw.DateField(),
)
migrator.change_fields(
"prompt",
timestamp=pw.DateField(),
)
migrator.change_fields(
"user",
timestamp=pw.DateField(),
)
migrator.change_fields(
"auth",
password=pw.CharField(max_length=255),
)
migrator.change_fields(
"chat",
title=pw.CharField(),
)
migrator.change_fields(
"document",
title=pw.CharField(),
filename=pw.CharField(),
)
migrator.change_fields(
"prompt",
title=pw.CharField(),
)
migrator.change_fields(
"user",
profile_image_url=pw.CharField(),
)
Original file line number Diff line number Diff line change
@@ -1,4 1,4 @@
"""Peewee migrations -- 009_add_models.py.
"""Peewee migrations -- 002_add_local_sharing.py.

Some examples (model - class or model name)::

Expand Down Expand Up @@ -37,25 37,43 @@
def migrate(migrator: Migrator, database: pw.Database, *, fake=False):
"""Write your migrations here."""

@migrator.create_model
class Model(pw.Model):
id = pw.TextField(unique=True)
user_id = pw.TextField()
base_model_id = pw.TextField(null=True)
# Adding fields created_at and updated_at to the 'user' table
migrator.add_fields(
"user",
created_at=pw.BigIntegerField(null=True), # Allow null for transition
updated_at=pw.BigIntegerField(null=True), # Allow null for transition
last_active_at=pw.BigIntegerField(null=True), # Allow null for transition
)

name = pw.TextField()
# Populate the new fields from an existing 'timestamp' field
migrator.sql(
'UPDATE "user" SET created_at = timestamp, updated_at = timestamp, last_active_at = timestamp WHERE timestamp IS NOT NULL'
)

meta = pw.TextField()
params = pw.TextField()
# Now that the data has been copied, remove the original 'timestamp' field
migrator.remove_fields("user", "timestamp")

created_at = pw.BigIntegerField(null=False)
updated_at = pw.BigIntegerField(null=False)

class Meta:
table_name = "model"
# Update the fields to be not null now that they are populated
migrator.change_fields(
"user",
created_at=pw.BigIntegerField(null=False),
updated_at=pw.BigIntegerField(null=False),
last_active_at=pw.BigIntegerField(null=False),
)


def rollback(migrator: Migrator, database: pw.Database, *, fake=False):
"""Write your rollback migrations here."""

migrator.remove_model("model")
# Recreate the timestamp field initially allowing null values for safe transition
migrator.add_fields("user", timestamp=pw.BigIntegerField(null=True))

# Copy the earliest created_at date back into the new timestamp field
# This assumes created_at was originally a copy of timestamp
migrator.sql('UPDATE "user" SET timestamp = created_at')

# Remove the created_at and updated_at fields
migrator.remove_fields("user", "created_at", "updated_at", "last_active_at")

# Finally, alter the timestamp field to not allow nulls if that was the original setting
migrator.change_fields("user", timestamp=pw.BigIntegerField(null=False))
24 changes: 8 additions & 16 deletions backend/apps/webui/internal/migrations/008_add_memory.py
Original file line number Diff line number Diff line change
@@ -1,4 1,4 @@
"""Peewee migrations -- 009_add_models.py.
"""Peewee migrations -- 002_add_local_sharing.py.

Some examples (model - class or model name)::

Expand Down Expand Up @@ -35,27 35,19 @@


def migrate(migrator: Migrator, database: pw.Database, *, fake=False):
"""Write your migrations here."""

@migrator.create_model
class Model(pw.Model):
id = pw.TextField(unique=True)
user_id = pw.TextField()
base_model_id = pw.TextField(null=True)

name = pw.TextField()

meta = pw.TextField()
params = pw.TextField()

created_at = pw.BigIntegerField(null=False)
class Memory(pw.Model):
id = pw.CharField(max_length=255, unique=True)
user_id = pw.CharField(max_length=255)
content = pw.TextField(null=False)
updated_at = pw.BigIntegerField(null=False)
created_at = pw.BigIntegerField(null=False)

class Meta:
table_name = "model"
table_name = "memory"


def rollback(migrator: Migrator, database: pw.Database, *, fake=False):
"""Write your rollback migrations here."""

migrator.remove_model("model")
migrator.remove_model("memory")
Loading
Loading