Initial commit

xmba15 · Jun 5, 2024 · cc0460c · cc0460c
commit cc0460c
Show file tree

Hide file tree

Showing 20 changed files with 758 additions and 0 deletions.
diff --git a/.flake8 b/.flake8
@@ -0,0  1,2 @@
 [flake8]
 max-line-length = 120
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -0,0  1,23 @@
 name: Build
 
 on:
   push:
     branches: ["master"]
   pull_request:
 
 jobs:
   linting:
     runs-on: ubuntu-20.04
 
     steps:
       - uses: actions/checkout@v3
 
       - name: Setup python
         uses: actions/setup-python@v4
         with:
           python-version: "3.8"
 
       - name: Apply pre-commit
         uses: pre-commit/[email protected]
         with:
           extra_args: --all-files
diff --git a/.gitignore b/.gitignore
@@ -0,0  1,88 @@
 # Compiled source #
 ###################
 *.com
 *.class
 *.dll
 *.exe
 *.o
 *.so
 *.pyc
 .ipynb_checkpoints
 *~
 *#
 build*
 
 # Packages #
 ###################
 # it's better to unpack these files and commit the raw source
 # git has its own built in compression methods
 *.7z
 *.dmg
 *.gz
 *.iso
 *.jar
 *.rar
 *.tar
 *.zip
 
 # Logs and databases #
 ######################
 *.log
 *.sql
 *.sqlite
 
 # OS generated files #
 ######################
 .DS_Store
 .DS_Store?
 ._*
 .Spotlight-V100
 .Trashes
 ehthumbs.db
 Thumbs.db
 
 # Images
 ######################
 *.jpg
 *.gif
 *.png
 *.svg
 *.ico
 
 # Video
 ######################
 *.wmv
 *.mpg
 *.mpeg
 *.mp4
 *.mov
 *.flv
 *.avi
 *.ogv
 *.ogg
 *.webm
 
 # Audio
 ######################
 *.wav
 *.mp3
 *.wma
 
 # Fonts
 ######################
 Fonts
 *.eot
 *.ttf
 *.woff
 
 # Format
 ######################
 CPPLINT.cfg
 .clang-format
 
 # Gtags
 ######################
 GPATH
 GRTAGS
 GSYMS
 GTAGS
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -0,0  1,44 @@
 repos:
   - repo: https://github.com/pycqa/flake8
     rev: 4.0.1
     hooks:
       - id: flake8
         additional_dependencies: [pep8-naming]
 
   - repo: https://github.com/psf/black
     rev: 22.6.0
     hooks:
       - id: black
         language_version: python3
 
   - repo: https://github.com/pycqa/isort
     rev: 5.12.0
     hooks:
       - id: isort
 
   - repo: https://github.com/pre-commit/mirrors-mypy
     rev: v0.961
     hooks:
       - id: mypy
 
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v4.3.0
     hooks:
       - id: end-of-file-fixer
       - id: trailing-whitespace
 
   - repo: https://github.com/pre-commit/mirrors-prettier
     rev: v2.7.1
     hooks:
       - id: prettier
         types_or: [json, markdown, yaml]
 
   - repo: https://github.com/lovesegfault/beautysh
     rev: v6.2.1
     hooks:
       - id: beautysh
 
   - repo: https://github.com/pylint-dev/pylint.git
     rev: v3.1.1
     hooks:
       - id: pylint
diff --git a/README.md b/README.md
@@ -0,0  1,132 @@
 # 📝 2D Thin Plate Spline Warping: Implementations & Experiments
 
 ---
 
 Simple samples to demonstrate 2D Thin Plate Spline Warping.
 
 The keypoint matches are obtained by lightglue matcher on KeyNetAffNetHardNet keypoints.
 
 ## :book: Explanation
 
 ---
 
 2D Thin Plate Spline (TPS) warping is a parametric transformation method to transform a source image to a target image.
 
 TPS mapping function $`\textbf{f}(\textbf{p})=\textbf{f}(x,y)`$ is estimated using control points with source points $`{\left\{ \textbf{p}_i=(x_i,y_i) \right\}}_{i=1}^N`$ in the source image that match with target points $`{\left\{ \textbf{p'}_i=(x'_i,y'_i) \right\}}_{i=1}^N`$.
 
 The loss function in 2D TPS warping is composed of two main components: data term and bending energy
 
 $$
 L=\sum_{i=1}^{N}\left\| f(\textbf{p}_i)-\textbf{p}'_i \right\|^2 \lambda\int\int\left[ \left( \frac{\partial ^2 f}{\partial x^2} \right)^2 2\left( \frac{\partial ^2f}{\partial x\partial y} \right)^2 \left(\frac{\partial ^2f}{\partial y^2}  \right)^2 \right]dxdy
 $$
 
 This optimization problem has a closed form solution as following:
 
 $$
 \textbf{f}(\textbf{p})=(f_x(x,y),f_y(x,y))
 $$
 
 $$
 f_x(x,y)=a_1 a_2x a_3y \sum_{i=1}^{N}w_iU(\left\| \textbf{p}-\textbf{p}_i \right\|)
 $$
 
 $$
 f_y(x,y)=b_1 b_2x b_3y \sum_{i=1}^{N}v_iU(\left\| \textbf{p}-\textbf{p}_i \right\|)
 $$
 
 where $`U(r)=r^2log(r^2)`$ is radial basis function, with $`r=\left\| \textbf{p}-\textbf{p}_i \right\|`$
 
 This system of equations has the following constraint:
 
 $$
 \sum_{i=1}^{N}w_i=\sum_{i=1}^{N}w_ix_i=\sum_{i=1}^{N}w_iy_i=0
 $$
 
 $$
 \sum_{i=1}^{N}v_i=\sum_{i=1}^{N}v_ix_i=\sum_{i=1}^{N}v_iy_i=0
 $$
 
 Hence, can be transformed to the matrix formulation $`\textbf{A}\textbf{c}=\textbf{b}`$
 
 $$
 A=\begin{bmatrix}
 \textbf{K} & \textbf{P}\\
 \textbf{P}^T & \textbf{0}
 \end{bmatrix}
 ,
 c=\begin{bmatrix}
 \textbf{w} \\
 \textbf{a}
 \end{bmatrix}
 ,
 b=\begin{bmatrix}
 \textbf{x}' \\
 \textbf{0}
 \end{bmatrix}
 $$
 
 $$
 K_{ij}=U(\left\| \textbf{p}_i-\textbf{p}_j \right\|);\textbf{K}\in \textbf{R}^{N \times N}
 $$
 
 $$
 \textbf{P}=\begin{bmatrix}
 1 & x_1 & y_1 \\
 1 & x_2 & y_2 \\
 ... & ... & ... \\
 1 & x_N & y_N
 \end{bmatrix}; \textbf{P} \in \textbf{R}^{N \times 3}
 $$
 
 $$
 \textbf{w}=[w_1w_2...w_N]^T;\textbf{a}=[a_1a_2a_3]^T;\textbf{x}'=[x_1'x_2'...x_N']^T
 $$
 
 ## :running: How to Run
 
 ---
 
 ```
 python3 scripts/test_tps.py --query_path ./data/car1.jpg --ref_path ./data/car2.jpg
 ```
 
 <table>
   <tr>
     <td align="center">
       <img src="./data/car1.jpg" alt="car1" width="45%" /><br />
       <b>query image</b>
     </td>
     <td align="center">
       <img src="./data/car2.jpg" alt="car2" width="45%" /><br />
       <b>reference image</b>
     </td>
   </tr>
 </table>
 
 <table>
   <tr>
     <td align="center">
       <img src="./docs/images/warped_image.jpg" alt="car1" width="45%" /><br />
       <b>warped query image</b>
     </td>
     <td align="center">
       <img src="./docs/images/merged_image.jpg" alt="car2" width="45%" /><br />
       <b>warped query image blended with reference</b>
     </td>
   </tr>
 </table>
 
 ## 🎛 Development environment
 
 ---
 
 ```bash
 mamba env create --file environment.yml
 mamba activate tps
 ```
 
 ## :gem: References
 
 ---
 
 - [Thin Plate Spline](https://en.wikipedia.org/wiki/Thin_plate_spline)
diff --git a/data/.keep b/data/.keep
diff --git a/data/car1.jpg b/data/car1.jpg
diff --git a/data/car2.jpg b/data/car2.jpg
diff --git a/docs/.keep b/docs/.keep
diff --git a/docs/images/merged_image.jpg b/docs/images/merged_image.jpg
diff --git a/docs/images/warped_image.jpg b/docs/images/warped_image.jpg
diff --git a/environment.yml b/environment.yml
@@ -0,0  1,8 @@
 name: tps
 channels:
   - defaults
 dependencies:
   - python=3.8
   - pip
   - pip:
       - -r requirements.txt
diff --git a/pyproject.toml b/pyproject.toml
@@ -0,0  1,19 @@
 [tool.black]
 line-length = 120
 
 [tool.isort]
 profile = "black"
 multi_line_output = 3
 
 [tool.mypy]
 ignore_missing_imports = true
 
 [tool.pylint."MESSAGES CONTROL"]
 disable = """
     missing-docstring,
     import-error,
     wrong-import-position,
 """
 max-line-length = 120
 
 ignore-paths = 'scripts/nms.py'
diff --git a/requirements.txt b/requirements.txt
@@ -0,0  1,3 @@
 opencv-python==4.1.2.30
 kornia==v0.7.2
 loguru==0.7.0
diff --git a/scripts/.keep b/scripts/.keep