Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Implement the AIASynopticClient #7842

Draft
wants to merge 22 commits into
base: main
Choose a base branch
from
Draft

Implement the AIASynopticClient #7842

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
22 commits
Select commit Hold shift click to select a range
ef8ed3f
hotfix the quantity comparison for wavelengths
GillySpace27 Oct 18, 2024
4fddb3c
update the __init__ files in dataretriever and sources
GillySpace27 Oct 18, 2024
43088b4
init AIASynopticClient in Sources
GillySpace27 Oct 18, 2024
2504767
add standard test file
GillySpace27 Oct 18, 2024
9095859
temporarily include bespoke test files
GillySpace27 Oct 18, 2024
d74cb27
tweak examples
GillySpace27 Oct 18, 2024
578b0f0
clean up synoptic client
GillySpace27 Oct 18, 2024
2548761
move import to top of file
GillySpace27 Oct 18, 2024
c801575
iteration savepoint
GillySpace27 Oct 21, 2024
3398c47
fix wavelength selection
GillySpace27 Oct 21, 2024
54ba715
first fix of sample
GillySpace27 Oct 21, 2024
247517a
simplify the manual test case
GillySpace27 Oct 22, 2024
8441345
extensively refactor the main client
GillySpace27 Oct 22, 2024
2bf7ade
clean notebook example
GillySpace27 Oct 22, 2024
20c5216
Delete manual testing scripts
GillySpace27 Oct 23, 2024
740e5d9
Dump sources/__init__.py content
GillySpace27 Oct 23, 2024
5c8c925
update required vs supported attrs
GillySpace27 Oct 23, 2024
dae0c15
Update sunpy/net/dataretriever/sources/tests/test_aia_synoptic.py
GillySpace27 Oct 24, 2024
a83a5a3
Update sunpy/net/dataretriever/sources/tests/test_aia_synoptic.py
GillySpace27 Oct 24, 2024
06f0873
Merge branch 'main' into AIASynopticSecondTry
GillySpace27 Nov 28, 2024
8859017
idk
GillySpace27 Jan 13, 2025
faccbf3
Merge branch 'AIASynopticSecondTry' of https://github.com/GillySpace2…
GillySpace27 Jan 13, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
temporarily include bespoke test files
  • Loading branch information
@GillySpace27
GillySpace27 committed Oct 18, 2024
commit 90958598e655b7c00dfbfdff9f964bd3c7c81898
137 changes: 137 additions & 0 deletions sunpy/net/dataretriever/sources/tests/example_aia_synoptic.ipynb
View file
Open in desktop
GillySpace27 marked this conversation as resolved.
Show resolved Hide resolved
Original file line number Diff line number Diff line change
@@ -0,0 1,137 @@
{
GillySpace27 marked this conversation as resolved.
Show resolved Hide resolved
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/Users/cgilbert/vscode/sunpy/sunpy/.venv/lib/python3.12/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
" from .autonotebook import tqdm as notebook_tqdm\n"
]
}
],
"source": [
"from sunpy.net import Fido\n",
"from sunpy.net import attrs as a\n",
"import astropy.units as u"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"WARNING: SunpyDeprecationWarning: pattern has been replaced with the format keyword. This comes with a new syntax and there is a migration guide available at https://docs.sunpy.org/en/stable/how_to/scraper_migration.html. [sunpy.net.dataretriever.client]\n",
"2024-10-18 13:21:33 - sunpy - WARNING: SunpyDeprecationWarning: pattern has been replaced with the format keyword. This comes with a new syntax and there is a migration guide available at https://docs.sunpy.org/en/stable/how_to/scraper_migration.html.\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Results from 1 Provider:\n",
"\n",
"12964 Results from the AIASynopticClient:\n",
"Source: https://jsoc1.stanford.edu/data/aia/synoptic/\n",
"\n",
" Start Time End Time ... Resolution Wavelength\n",
"----------------------- ----------------------- ... ---------- ----------\n",
"2013-10-30 00:00:00.000 2013-10-30 00:00:59.999 ... SYNOPTIC 94\n",
"2013-10-30 00:00:00.000 2013-10-30 00:00:59.999 ... SYNOPTIC 131\n",
"2013-10-30 00:00:00.000 2013-10-30 00:00:59.999 ... SYNOPTIC 171\n",
"2013-10-30 00:00:00.000 2013-10-30 00:00:59.999 ... SYNOPTIC 193\n",
"2013-10-30 00:00:00.000 2013-10-30 00:00:59.999 ... SYNOPTIC 211\n",
"2013-10-30 00:00:00.000 2013-10-30 00:00:59.999 ... SYNOPTIC 304\n",
"2013-10-30 00:00:00.000 2013-10-30 00:00:59.999 ... SYNOPTIC 335\n",
"2013-10-30 00:00:00.000 2013-10-30 00:00:59.999 ... SYNOPTIC 1600\n",
"2013-10-30 00:00:00.000 2013-10-30 00:00:59.999 ... SYNOPTIC 4500\n",
"2013-10-30 00:02:00.000 2013-10-30 00:02:59.999 ... SYNOPTIC 94\n",
" ... ... ... ... ...\n",
"2013-10-31 23:58:00.000 2013-10-31 23:58:59.999 ... SYNOPTIC 1700\n",
"2013-11-01 00:00:00.000 2013-11-01 00:00:59.999 ... SYNOPTIC 94\n",
"2013-11-01 00:00:00.000 2013-11-01 00:00:59.999 ... SYNOPTIC 131\n",
"2013-11-01 00:00:00.000 2013-11-01 00:00:59.999 ... SYNOPTIC 171\n",
"2013-11-01 00:00:00.000 2013-11-01 00:00:59.999 ... SYNOPTIC 193\n",
"2013-11-01 00:00:00.000 2013-11-01 00:00:59.999 ... SYNOPTIC 211\n",
"2013-11-01 00:00:00.000 2013-11-01 00:00:59.999 ... SYNOPTIC 304\n",
"2013-11-01 00:00:00.000 2013-11-01 00:00:59.999 ... SYNOPTIC 335\n",
"2013-11-01 00:00:00.000 2013-11-01 00:00:59.999 ... SYNOPTIC 1600\n",
"2013-11-01 00:00:00.000 2013-11-01 00:00:59.999 ... SYNOPTIC 4500\n",
"Length = 12964 rows\n",
"\n",
"\n"
]
}
],
"source": [
"# Example usage\n",
"time_range = a.Time(\"2013-10-30 00:00:00\", \"2013-11-01 00:00:00\")\n",
"instrument = a.Instrument(\"AIASynoptic\")\n",
"wavelength = a.Wavelength(171 * u.angstrom)\n",
"# sample = a.Sample(10 * u.minute)\n",
"results = Fido.search(time_range, instrument, wavelength) # , wavelength)\n",
"\n",
"print(results)\n",
"\n"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Downloading to: /Users/cgilbert/vscode/DLData/Halloween/{file}\n"
]
}
],
"source": [
"import os\n",
"the_path = \"~/vscode/DLData/Halloween/{file}\"\n",
"the_path = os.path.expanduser(the_path)\n",
"print(f\"Downloading to: {the_path}\")\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"Fido.fetch(results, path=the_path)"
]
}
],
"metadata": {
"kernelspec": {
"display_name": ".venv",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.12.6"
}
},
"nbformat": 4,
"nbformat_minor": 2
}
74 changes: 74 additions & 0 deletions sunpy/net/dataretriever/sources/tests/synoptic_manual_test.py
View file
Open in desktop
GillySpace27 marked this conversation as resolved.
Show resolved Hide resolved
Original file line number Diff line number Diff line change
@@ -0,0 1,74 @@
# from sunpy.net import attrs as a, Fido
GillySpace27 marked this conversation as resolved.
Show resolved Hide resolved
# import astropy.units as u

# req = a.Time("2020/01/01 00:00:00", "2020/01/05 00:00:00")
# found = Fido.search(req, a.Instrument("AIASynoptic"), a.Sample(10 * u.minute))
# print(found)
# import re
# from urllib.parse import urlparse

# # info_url = r"https://jsoc1.stanford.edu/data/aia/synoptic/"
# # baseurl = info_url r"%Y/%m/%d/H%H00/AIA%Y%m%d_%H%M_.....fits"
# # extractor = pattern = info_url r"%Y/%m/%d/H%H00/AIA%Y%m%d_%H%M_{}.fits"

# # Assuming you have the necessary imports and setup
# extractor = r"https://jsoc1.stanford.edu/data/aia/synoptic/(\d{4})/(\d{2})/(\d{2})/H(\d{2})00/AIA(?P<year>\d{4})(?P<month>\d{2})(?P<day>\d{2})_(?P<hour>\d{2})(?P<minute>\d{2})_(?P<wavelength>\d{4})\.fits"
# print(extractor)
# # Example URL
# url = "https://jsoc1.stanford.edu/data/aia/synoptic/2023/10/11/H0300/AIA20231011_0300_1234.fits"

# # Parse the URL using regex
# matches = re.match(extractor, url)

# # Check if there's a match and extract named groups
# if matches:
# exdict = matches.groupdict() # This will give you a dictionary of named groups
# print("Match found! Extracted values:", exdict)
# else:
# print("No match found.")


# import re

# url = 'https://jsoc1.stanford.edu/data/aia/synoptic/2023/10/11/H0000/AIA20231011_0000_0094.fits'
# regex = r'https://jsoc1.stanford.edu/data/aia/synoptic/\d{4}/\d{2}/\d{2}/H\d{2}00/AIA\d{8}_\d{4}_\d \.fits'

# if re.match(regex, url):
# print("Match!")
# else:
# print("No match.")


# from sunpy.net import Fido
# from sunpy.net import attrs as a
# import astropy.units as u


# # Example usage
# time_range = a.Time("2023-10-11 00:00:00", "2023-10-11 01:00:00")
# instrument = a.Instrument("AIASynoptic")
# wavelength = a.Wavelength(193 * u.angstrom)
# sample = a.Sample(10 * u.minute)
# results = Fido.search(time_range, instrument, wavelength) # , sample) # , wavelength)

# print(results)

# Fido.fetch(results, path="DLData/{file}")


if __name__ == "__main__":
from sunpy.net import Fido
from sunpy.net import attrs as a
import astropy.units as u

# Example usage
time_range = a.Time("2023-10-11 00:00:00", "2023-10-11 01:00:00")
instrument = a.Instrument("AIA")
wavelength = a.Wavelength(193 * u.angstrom)
resolution = a.Resolution("Synoptic")
# sample = a.Sample(10 * u.minute)
results = Fido.search(time_range, instrument, wavelength, resolution) # , sample) # , wavelength)

print(results)

# Fido.fetch(results, path="DLData/{file}")