Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: db migration #3595

Merged
merged 34 commits into from
Jul 6, 2024
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
34 commits
Select commit Hold shift click to select a range
df09d08
feat(sqlalchemy): Replace peewee with sqlalchemy
jonathan-rohde Jun 18, 2024
bee835c
feat(sqlalchemy): remove session reference from router
jonathan-rohde Jun 21, 2024
070d908
feat(sqlalchemy): use subprocess to do migrations
jonathan-rohde Jun 24, 2024
320e658
feat(sqlalchemy): cleanup fixes
jonathan-rohde Jun 24, 2024
c134eab
feat(sqlalchemy): format backend
jonathan-rohde Jun 24, 2024
eb01e8d
feat(sqlalchemy): use scoped session
jonathan-rohde Jun 24, 2024
da403f3
feat(sqlalchemy): use session factory instead of context manager
jonathan-rohde Jun 24, 2024
a9b1487
feat(sqlalchemy): fix wrong column types
jonathan-rohde Jun 24, 2024
8f939cf
feat(sqlalchemy): some fixes
jonathan-rohde Jun 24, 2024
2fb27ad
feat(sqlalchemy): add missing file
jonathan-rohde Jun 24, 2024
d88bd51
feat(sqlalchemy): format backend
jonathan-rohde Jun 24, 2024
642c352
feat(sqlalchemy): rebase
jonathan-rohde Jun 25, 2024
d4b6b7c
feat(sqlalchemy): reverted not needed api change
jonathan-rohde Jun 25, 2024
23e4d9d
feat(sqlalchemy): formatting
jonathan-rohde Jun 25, 2024
827b1e5
feat(sqlalchemy): execute tests in github actions
jonathan-rohde Jun 25, 2024
df47c49
Merge branch 'refs/heads/dev' into feat/sqlalchemy-instead-of-peewee
jonathan-rohde Jun 28, 2024
5391f4c
feat(sqlalchemy): add new column
jonathan-rohde Jun 28, 2024
2aecd7d
Merge branch 'refs/heads/dev' into feat/sqlalchemy-instead-of-peewee
jonathan-rohde Jul 1, 2024
d0e89a0
Merge pull request #3327 from jonathan-rohde/feat/sqlalchemy-instead-…
tjbck Jul 2, 2024
647aa19
chore: format
tjbck Jul 2, 2024
44a9b86
fix: functions
tjbck Jul 3, 2024
aa88022
fix: functions
tjbck Jul 3, 2024
4d23957
revert: model_validate
tjbck Jul 3, 2024
0d78b63
Merge pull request #3621 from open-webui/dev
tjbck Jul 4, 2024
15f6f7b
revert: peewee migrations
tjbck Jul 4, 2024
bfc53b4
revert
tjbck Jul 4, 2024
1b65df3
revert
tjbck Jul 4, 2024
8646460
refac
tjbck Jul 4, 2024
37a5d2c
Update db.py
tjbck Jul 4, 2024
8fe2a7b
fix
tjbck Jul 4, 2024
8b13755
Update auths.py
tjbck Jul 4, 2024
d60f066
Merge pull request #3668 from open-webui/dev
tjbck Jul 6, 2024
1436bb7
enh: handle peewee migration
tjbck Jul 6, 2024
4e75150
Merge pull request #3669 from open-webui/dev-migration-session
tjbck Jul 6, 2024
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
revert: peewee migrations
  • Loading branch information
tjbck committed Jul 4, 2024
commit 15f6f7bd15b642e422f47f8d97b2056f043b15b2
254 changes: 254 additions & 0 deletions backend/apps/webui/internal/migrations/001_initial_schema.py
Original file line number Diff line number Diff line change
@@ -0,0 1,254 @@
"""Peewee migrations -- 001_initial_schema.py.

Some examples (model - class or model name)::

> Model = migrator.orm['table_name'] # Return model in current state by name
> Model = migrator.ModelClass # Return model in current state by name

> migrator.sql(sql) # Run custom SQL
> migrator.run(func, *args, **kwargs) # Run python function with the given args
> migrator.create_model(Model) # Create a model (could be used as decorator)
> migrator.remove_model(model, cascade=True) # Remove a model
> migrator.add_fields(model, **fields) # Add fields to a model
> migrator.change_fields(model, **fields) # Change fields
> migrator.remove_fields(model, *field_names, cascade=True)
> migrator.rename_field(model, old_field_name, new_field_name)
> migrator.rename_table(model, new_table_name)
> migrator.add_index(model, *col_names, unique=False)
> migrator.add_not_null(model, *field_names)
> migrator.add_default(model, field_name, default)
> migrator.add_constraint(model, name, sql)
> migrator.drop_index(model, *col_names)
> migrator.drop_not_null(model, *field_names)
> migrator.drop_constraints(model, *constraints)

"""

from contextlib import suppress

import peewee as pw
from peewee_migrate import Migrator


with suppress(ImportError):
import playhouse.postgres_ext as pw_pext


def migrate(migrator: Migrator, database: pw.Database, *, fake=False):
"""Write your migrations here."""

# We perform different migrations for SQLite and other databases
# This is because SQLite is very loose with enforcing its schema, and trying to migrate other databases like SQLite
# will require per-database SQL queries.
# Instead, we assume that because external DB support was added at a later date, it is safe to assume a newer base
# schema instead of trying to migrate from an older schema.
if isinstance(database, pw.SqliteDatabase):
migrate_sqlite(migrator, database, fake=fake)
else:
migrate_external(migrator, database, fake=fake)


def migrate_sqlite(migrator: Migrator, database: pw.Database, *, fake=False):
@migrator.create_model
class Auth(pw.Model):
id = pw.CharField(max_length=255, unique=True)
email = pw.CharField(max_length=255)
password = pw.CharField(max_length=255)
active = pw.BooleanField()

class Meta:
table_name = "auth"

@migrator.create_model
class Chat(pw.Model):
id = pw.CharField(max_length=255, unique=True)
user_id = pw.CharField(max_length=255)
title = pw.CharField()
chat = pw.TextField()
timestamp = pw.BigIntegerField()

class Meta:
table_name = "chat"

@migrator.create_model
class ChatIdTag(pw.Model):
id = pw.CharField(max_length=255, unique=True)
tag_name = pw.CharField(max_length=255)
chat_id = pw.CharField(max_length=255)
user_id = pw.CharField(max_length=255)
timestamp = pw.BigIntegerField()

class Meta:
table_name = "chatidtag"

@migrator.create_model
class Document(pw.Model):
id = pw.AutoField()
collection_name = pw.CharField(max_length=255, unique=True)
name = pw.CharField(max_length=255, unique=True)
title = pw.CharField()
filename = pw.CharField()
content = pw.TextField(null=True)
user_id = pw.CharField(max_length=255)
timestamp = pw.BigIntegerField()

class Meta:
table_name = "document"

@migrator.create_model
class Modelfile(pw.Model):
id = pw.AutoField()
tag_name = pw.CharField(max_length=255, unique=True)
user_id = pw.CharField(max_length=255)
modelfile = pw.TextField()
timestamp = pw.BigIntegerField()

class Meta:
table_name = "modelfile"

@migrator.create_model
class Prompt(pw.Model):
id = pw.AutoField()
command = pw.CharField(max_length=255, unique=True)
user_id = pw.CharField(max_length=255)
title = pw.CharField()
content = pw.TextField()
timestamp = pw.BigIntegerField()

class Meta:
table_name = "prompt"

@migrator.create_model
class Tag(pw.Model):
id = pw.CharField(max_length=255, unique=True)
name = pw.CharField(max_length=255)
user_id = pw.CharField(max_length=255)
data = pw.TextField(null=True)

class Meta:
table_name = "tag"

@migrator.create_model
class User(pw.Model):
id = pw.CharField(max_length=255, unique=True)
name = pw.CharField(max_length=255)
email = pw.CharField(max_length=255)
role = pw.CharField(max_length=255)
profile_image_url = pw.CharField(max_length=255)
timestamp = pw.BigIntegerField()

class Meta:
table_name = "user"


def migrate_external(migrator: Migrator, database: pw.Database, *, fake=False):
@migrator.create_model
class Auth(pw.Model):
id = pw.CharField(max_length=255, unique=True)
email = pw.CharField(max_length=255)
password = pw.TextField()
active = pw.BooleanField()

class Meta:
table_name = "auth"

@migrator.create_model
class Chat(pw.Model):
id = pw.CharField(max_length=255, unique=True)
user_id = pw.CharField(max_length=255)
title = pw.TextField()
chat = pw.TextField()
timestamp = pw.BigIntegerField()

class Meta:
table_name = "chat"

@migrator.create_model
class ChatIdTag(pw.Model):
id = pw.CharField(max_length=255, unique=True)
tag_name = pw.CharField(max_length=255)
chat_id = pw.CharField(max_length=255)
user_id = pw.CharField(max_length=255)
timestamp = pw.BigIntegerField()

class Meta:
table_name = "chatidtag"

@migrator.create_model
class Document(pw.Model):
id = pw.AutoField()
collection_name = pw.CharField(max_length=255, unique=True)
name = pw.CharField(max_length=255, unique=True)
title = pw.TextField()
filename = pw.TextField()
content = pw.TextField(null=True)
user_id = pw.CharField(max_length=255)
timestamp = pw.BigIntegerField()

class Meta:
table_name = "document"

@migrator.create_model
class Modelfile(pw.Model):
id = pw.AutoField()
tag_name = pw.CharField(max_length=255, unique=True)
user_id = pw.CharField(max_length=255)
modelfile = pw.TextField()
timestamp = pw.BigIntegerField()

class Meta:
table_name = "modelfile"

@migrator.create_model
class Prompt(pw.Model):
id = pw.AutoField()
command = pw.CharField(max_length=255, unique=True)
user_id = pw.CharField(max_length=255)
title = pw.TextField()
content = pw.TextField()
timestamp = pw.BigIntegerField()

class Meta:
table_name = "prompt"

@migrator.create_model
class Tag(pw.Model):
id = pw.CharField(max_length=255, unique=True)
name = pw.CharField(max_length=255)
user_id = pw.CharField(max_length=255)
data = pw.TextField(null=True)

class Meta:
table_name = "tag"

@migrator.create_model
class User(pw.Model):
id = pw.CharField(max_length=255, unique=True)
name = pw.CharField(max_length=255)
email = pw.CharField(max_length=255)
role = pw.CharField(max_length=255)
profile_image_url = pw.TextField()
timestamp = pw.BigIntegerField()

class Meta:
table_name = "user"


def rollback(migrator: Migrator, database: pw.Database, *, fake=False):
"""Write your rollback migrations here."""

migrator.remove_model("user")

migrator.remove_model("tag")

migrator.remove_model("prompt")

migrator.remove_model("modelfile")

migrator.remove_model("document")

migrator.remove_model("chatidtag")

migrator.remove_model("chat")

migrator.remove_model("auth")
48 changes: 48 additions & 0 deletions backend/apps/webui/internal/migrations/002_add_local_sharing.py
Original file line number Diff line number Diff line change
@@ -0,0 1,48 @@
"""Peewee migrations -- 002_add_local_sharing.py.

Some examples (model - class or model name)::

> Model = migrator.orm['table_name'] # Return model in current state by name
> Model = migrator.ModelClass # Return model in current state by name

> migrator.sql(sql) # Run custom SQL
> migrator.run(func, *args, **kwargs) # Run python function with the given args
> migrator.create_model(Model) # Create a model (could be used as decorator)
> migrator.remove_model(model, cascade=True) # Remove a model
> migrator.add_fields(model, **fields) # Add fields to a model
> migrator.change_fields(model, **fields) # Change fields
> migrator.remove_fields(model, *field_names, cascade=True)
> migrator.rename_field(model, old_field_name, new_field_name)
> migrator.rename_table(model, new_table_name)
> migrator.add_index(model, *col_names, unique=False)
> migrator.add_not_null(model, *field_names)
> migrator.add_default(model, field_name, default)
> migrator.add_constraint(model, name, sql)
> migrator.drop_index(model, *col_names)
> migrator.drop_not_null(model, *field_names)
> migrator.drop_constraints(model, *constraints)

"""

from contextlib import suppress

import peewee as pw
from peewee_migrate import Migrator


with suppress(ImportError):
import playhouse.postgres_ext as pw_pext


def migrate(migrator: Migrator, database: pw.Database, *, fake=False):
"""Write your migrations here."""

migrator.add_fields(
"chat", share_id=pw.CharField(max_length=255, null=True, unique=True)
)


def rollback(migrator: Migrator, database: pw.Database, *, fake=False):
"""Write your rollback migrations here."""

migrator.remove_fields("chat", "share_id")
48 changes: 48 additions & 0 deletions backend/apps/webui/internal/migrations/003_add_auth_api_key.py
Original file line number Diff line number Diff line change
@@ -0,0 1,48 @@
"""Peewee migrations -- 002_add_local_sharing.py.

Some examples (model - class or model name)::

> Model = migrator.orm['table_name'] # Return model in current state by name
> Model = migrator.ModelClass # Return model in current state by name

> migrator.sql(sql) # Run custom SQL
> migrator.run(func, *args, **kwargs) # Run python function with the given args
> migrator.create_model(Model) # Create a model (could be used as decorator)
> migrator.remove_model(model, cascade=True) # Remove a model
> migrator.add_fields(model, **fields) # Add fields to a model
> migrator.change_fields(model, **fields) # Change fields
> migrator.remove_fields(model, *field_names, cascade=True)
> migrator.rename_field(model, old_field_name, new_field_name)
> migrator.rename_table(model, new_table_name)
> migrator.add_index(model, *col_names, unique=False)
> migrator.add_not_null(model, *field_names)
> migrator.add_default(model, field_name, default)
> migrator.add_constraint(model, name, sql)
> migrator.drop_index(model, *col_names)
> migrator.drop_not_null(model, *field_names)
> migrator.drop_constraints(model, *constraints)

"""

from contextlib import suppress

import peewee as pw
from peewee_migrate import Migrator


with suppress(ImportError):
import playhouse.postgres_ext as pw_pext


def migrate(migrator: Migrator, database: pw.Database, *, fake=False):
"""Write your migrations here."""

migrator.add_fields(
"user", api_key=pw.CharField(max_length=255, null=True, unique=True)
)


def rollback(migrator: Migrator, database: pw.Database, *, fake=False):
"""Write your rollback migrations here."""

migrator.remove_fields("user", "api_key")
Loading
Loading