Merge pull request #97 from Congress-Dev/master-tagging

Add new schema for prompts
Congress-Dev · Dec 21, 2024 · 414c811 · 414c811
2 parents 8ff8286 174d7f0
commit 414c811
Show file tree

Hide file tree

Showing 18 changed files with 473 additions and 24 deletions.
diff --git a/.docker/docker-compose.prod.yml b/.docker/docker-compose.prod.yml
@@ -6,15  6,15 @@ services:
     - db_host=10.0.0.248:5432
     - CACHE_HEADER_TIME=432000
     - CACHE_TIME=900
-    - db_table=us_code
     - db_table=us_code_2023
     restart: unless-stopped
   congress_parser_fastapi:
     environment:
     - STAGE=prod
     - db_host=10.0.0.248:5432
     - CACHE_HEADER_TIME=432000
     - CACHE_TIME=900
-    - db_table=us_code
     - db_table=us_code_2023
     restart: unless-stopped
   congress_viewer_app:
     volumes:

diff --git a/backend/alembic/env.py b/backend/alembic/env.py
@@ -5,7  5,7 @@
 
 from alembic import context
 
-from billparser.db.models import Base, AppropriationsBase
 from billparser.db.models import Base, AppropriationsBase, PromptsBase
 
 # this is the Alembic Config object, which provides
 # access to the values within the .ini file in use.
@@ -30,6  30,7 @@ def merge_metadata(*original_metadata) -> MetaData:
 # from myapp import mymodel
 # target_metadata = mymodel.Base.metadata
 target_metadata = merge_metadata(Base.metadata, AppropriationsBase.metadata)
 target_metadata = merge_metadata(target_metadata, PromptsBase.metadata)
 # other values from the config, defined by the needs of env.py,
 # can be acquired:
 # my_important_option = config.get_main_option("my_important_option")

diff --git a/backend/alembic/versions/9a72e5faa767_add_prompts_schema.py b/backend/alembic/versions/9a72e5faa767_add_prompts_schema.py
@@ -0,0  1,60 @@
 """Add prompts schema
 
 Revision ID: 9a72e5faa767
 Revises: 1d773a33ec53
 Create Date: 2024-12-18 21:02:06.298753
 
 """
 from typing import Sequence, Union
 
 from alembic import op
 import sqlalchemy as sa
 from sqlalchemy.dialects import postgresql
 from sqlalchemy.schema import CreateSchema
 
 # revision identifiers, used by Alembic.
 revision: str = '9a72e5faa767'
 down_revision: Union[str, None] = '1d773a33ec53'
 branch_labels: Union[str, Sequence[str], None] = None
 depends_on: Union[str, Sequence[str], None] = None
 
 
 def upgrade() -> None:
     op.execute(CreateSchema("prompts", if_not_exists=True))
     # ### commands auto generated by Alembic - please adjust! ###
     op.create_table('prompt',
     sa.Column('prompt_id', sa.Integer(), nullable=False),
     sa.Column('version', sa.String(), nullable=False),
     sa.Column('title', sa.String(), nullable=False),
     sa.Column('description', sa.String(), nullable=True),
     sa.Column('prompt', sa.String(), nullable=False),
     sa.PrimaryKeyConstraint('prompt_id'),
     schema='prompts'
     )
     op.create_table('prompt_batch',
     sa.Column('prompt_batch_id', sa.Integer(), nullable=False),
     sa.Column('prompt_id', sa.Integer(), nullable=True),
     sa.Column('legislation_version_id', sa.Integer(), nullable=True),
     sa.Column('created_at', sa.DateTime(), server_default=sa.text('now()'), nullable=True),
     sa.Column('completed_at', sa.DateTime(), nullable=True),
     sa.Column('attempted', sa.Integer(), nullable=False),
     sa.Column('successful', sa.Integer(), nullable=False),
     sa.Column('failed', sa.Integer(), nullable=False),
     sa.Column('skipped', sa.Integer(), nullable=False),
     sa.ForeignKeyConstraint(['legislation_version_id'], ['legislation_version.legislation_version_id'], ondelete='CASCADE'),
     sa.ForeignKeyConstraint(['prompt_id'], ['prompts.prompt.prompt_id'], ondelete='CASCADE'),
     sa.PrimaryKeyConstraint('prompt_batch_id'),
     schema='prompts'
     )
     op.create_index(op.f('ix_prompts_prompt_batch_legislation_version_id'), 'prompt_batch', ['legislation_version_id'], unique=False, schema='prompts')
     op.create_index(op.f('ix_prompts_prompt_batch_prompt_id'), 'prompt_batch', ['prompt_id'], unique=False, schema='prompts')
     # ### end Alembic commands ###
 
 
 def downgrade() -> None:
     # ### commands auto generated by Alembic - please adjust! ###
     op.drop_index(op.f('ix_prompts_prompt_batch_prompt_id'), table_name='prompt_batch', schema='prompts')
     op.drop_index(op.f('ix_prompts_prompt_batch_legislation_version_id'), table_name='prompt_batch', schema='prompts')
     op.drop_table('prompt_batch', schema='prompts')
     op.drop_table('prompt', schema='prompts')
     # ### end Alembic commands ###
diff --git a/backend/alembic/versions/aee414b2c361_create_prompt_tagging_table.py b/backend/alembic/versions/aee414b2c361_create_prompt_tagging_table.py
@@ -0,0  1,44 @@
 """Create prompt tagging table
 
 Revision ID: aee414b2c361
 Revises: 9a72e5faa767
 Create Date: 2024-12-18 21:04:37.411800
 
 """
 from typing import Sequence, Union
 
 from alembic import op
 import sqlalchemy as sa
 from sqlalchemy.dialects import postgresql
 
 # revision identifiers, used by Alembic.
 revision: str = 'aee414b2c361'
 down_revision: Union[str, None] = '9a72e5faa767'
 branch_labels: Union[str, Sequence[str], None] = None
 depends_on: Union[str, Sequence[str], None] = None
 
 
 def upgrade() -> None:
     # ### commands auto generated by Alembic - please adjust! ###
     op.create_table('legislation_content_tag',
     sa.Column('legislation_content_tag_id', sa.Integer(), nullable=False),
     sa.Column('prompt_batch_id', sa.Integer(), nullable=True),
     sa.Column('legislation_content_id', sa.Integer(), nullable=True),
     sa.Column('tags', postgresql.ARRAY(sa.String()), nullable=True),
     sa.ForeignKeyConstraint(['legislation_content_id'], ['legislation_content.legislation_content_id'], ondelete='CASCADE'),
     sa.ForeignKeyConstraint(['prompt_batch_id'], ['prompts.prompt_batch.prompt_batch_id'], ),
     sa.PrimaryKeyConstraint('legislation_content_tag_id')
     )
     op.create_index(op.f('ix_legislation_content_tag_legislation_content_id'), 'legislation_content_tag', ['legislation_content_id'], unique=False)
     op.create_index(op.f('ix_legislation_content_tag_prompt_batch_id'), 'legislation_content_tag', ['prompt_batch_id'], unique=False)
     op.create_index(op.f('ix_legislation_content_tag_tags'), 'legislation_content_tag', ['tags'], unique=False)
     # ### end Alembic commands ###
 
 
 def downgrade() -> None:
     # ### commands auto generated by Alembic - please adjust! ###
     op.drop_index(op.f('ix_legislation_content_tag_tags'), table_name='legislation_content_tag')
     op.drop_index(op.f('ix_legislation_content_tag_prompt_batch_id'), table_name='legislation_content_tag')
     op.drop_index(op.f('ix_legislation_content_tag_legislation_content_id'), table_name='legislation_content_tag')
     op.drop_table('legislation_content_tag')
     # ### end Alembic commands ###
diff --git a/backend/billparser/db/models.py b/backend/billparser/db/models.py
@@ -25,6  25,7 @@
 
 # This will hold various things related to my appropriation parsing
 AppropriationsBase = declarative_base(metadata=MetaData(schema="appropriations"))
 PromptsBase = declarative_base(metadata=MetaData(schema="prompts"))
 
 
 class CastingArray(ARRAY):
@@ -156,6  157,19 @@ def to_dict(self):
         }
         return {k: v for (k, v) in boi.items() if v is not None}
 
 class Prompt(PromptsBase):
     """
     Table for holding the prompts
     """
 
     __tablename__ = "prompt"
     __table_args__ = {'schema': 'prompts'}
     prompt_id = Column(Integer, primary_key=True)
     version = Column(String, nullable=False)
     title = Column(String, nullable=False)
     description = Column(String, nullable=True)
     prompt = Column(String, nullable=False)
 
 
 class Legislation(Base):
     """
@@ -240,7  254,35 @@ def to_dict(self):
 except:
     pass
 
 class PromptBatch(PromptsBase):
     """
     Table for holding the records for which legislation versions have run through a prompt
     """
 
     __tablename__ = "prompt_batch"
     __table_args__ = {'schema': 'prompts'}
     prompt_batch_id = Column(Integer, primary_key=True)
     prompt_id = Column(
         Integer,
         ForeignKey(Prompt.prompt_id, ondelete="CASCADE"),
         index=True,
     )
     legislation_version_id = Column(
         Integer,
         ForeignKey(LegislationVersion.legislation_version_id, ondelete="CASCADE"),
         index=True,
     )
 
     created_at = Column(DateTime(timezone=False), server_default=func.now())
     completed_at = Column(DateTime(timezone=False), nullable=True)
 
     # Unique to each prompt, but holds info about how many items in the bill have been attempted
     attempted = Column(Integer, nullable=False, default=0)
     successful = Column(Integer, nullable=False, default=0)  # Generated a response
     failed = Column(Integer, nullable=False, default=0)  # Failed to generate a response
     skipped = Column(
         Integer, nullable=False, default=0
     )  # Skipped due to not matching the predicate
 class LegislationContent(Base):
     __tablename__ = "legislation_content"
 
@@ -289,6  331,25 @@ def to_dict(self):
         return {k: v for (k, v) in boi.items() if v is not None and v != {}}
 
 
 class LegislationContentTag(Base):
     """
     Represents a tag on a piece of legislation content
     """
 
     __tablename__ = "legislation_content_tag"
 
     legislation_content_tag_id = Column(Integer, primary_key=True)
     prompt_batch_id = Column(
         Integer, ForeignKey(PromptBatch.prompt_batch_id), index=True, nullable=True
     )
     legislation_content_id = Column(
         Integer,
         ForeignKey("legislation_content.legislation_content_id", ondelete="CASCADE"),
         index=True,
     )
     tags = Column(ARRAY(String), index=True)
 
 
 class USCRelease(Base):
     """
     Represents a release point of the USCode, as described by the prior release points page
@@ -746,36  807,38 @@ class LegislativePolicyAreaAssociation(Base):
         ForeignKey("legislation.legislation_id", ondelete="CASCADE"),
         index=True,
     )
 
 
 def merge_metadata(*metadata):
     merged = MetaData()
     for metadatum in metadata:
         for table in metadatum.tables.values():
             table.to_metadata(merged)
     return merged
 
 
 merge_metadata(Base.metadata, AppropriationsBase.metadata)
 
 
 class Appropriation(AppropriationsBase):
     """
     A table for holding detected appropriations
     """
 
     __tablename__ = "appropriation"
-    
 
     appropriation_id = Column(Integer, primary_key=True)
     parent_id = Column(Integer, nullable=True)
     legislation_version_id = Column(
         Integer,
-        ForeignKey(
-            LegislationVersion.legislation_version_id, ondelete="CASCADE"
-        ),
         ForeignKey(LegislationVersion.legislation_version_id, ondelete="CASCADE"),
         index=True,
     )
 
     # This is what we'll mainly use to actually key it off
     legislation_content_id = Column(
         Integer,
-        ForeignKey(
-            LegislationContent.legislation_content_id, ondelete="CASCADE"
-        ),
         ForeignKey(LegislationContent.legislation_content_id, ondelete="CASCADE"),
         index=True,
     )
     # For text highlighting
@@ -792,3  855,5 @@ class Appropriation(AppropriationsBase):
     target = Column(String, nullable=True)
 
     purpose = Column(String, default="")
 
 merge_metadata(Base.metadata, PromptsBase.metadata)
diff --git a/backend/billparser/prompt_runners/__init__.py b/backend/billparser/prompt_runners/__init__.py