theluyuan/shynet
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add Black config and pre-commit hook (#231)

Browse Source 
* Add black config to pyproject.toml

* Add flake8 config

* Add pre-commit black hook
This commit is contained in:
havk 2022-09-29 01:16:40 +02:00 committed by GitHub
parent e507dd5814
commit ba2f47edb7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 124 additions and 71 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.flake8 Normal file
View File

@ -0,0 +1,2 @@
[flake8]
max-line-length = 88

16
.pre-commit-config.yaml Normal file
View File

@ -0,0 +1,16 @@
# See https://pre-commit.com for more information
# See https://pre-commit.com/hooks.html for more hooks
repos:
#- repo: https://github.com/pre-commit/pre-commit-hooks
#rev: v3.2.0
#hooks:
#- id: trailing-whitespace
#- id: end-of-file-fixer
#- id: check-yaml
#- id: check-added-large-files
- repo: https://github.com/psf/black
rev: 22.8.0
hooks:
- id: black
exclude: 'migrations|^shynet/shynet/settings.py'

3
pyproject.toml
View File

@ -39,3 +39,6 @@ mypy = "^0.910"
[build-system] [build-system]
requires = ["poetry-core>=1.0.0"] requires = ["poetry-core>=1.0.0"]
build-backend = "poetry.core.masonry.api" build-backend = "poetry.core.masonry.api"
[tool.black]
line-length = 88

47
shynet/analytics/models.py
View File

@ -15,27 +15,26 @@ def _default_uuid():
class Session(models.Model): class Session(models.Model):
uuid = models.UUIDField(default=_default_uuid, primary_key=True) uuid = models.UUIDField(default=_default_uuid, primary_key=True)
service = models.ForeignKey( service = models.ForeignKey(
Service, verbose_name=_('Service'), Service, verbose_name=_("Service"), on_delete=models.CASCADE, db_index=True
on_delete=models.CASCADE, db_index=True
) )
# Cross-session identification; optional, and provided by the service # Cross-session identification; optional, and provided by the service
identifier = models.TextField( identifier = models.TextField(
blank=True, db_index=True, verbose_name=_('Identifier') blank=True, db_index=True, verbose_name=_("Identifier")
) )
# Time # Time
start_time = models.DateTimeField( start_time = models.DateTimeField(
default=timezone.now, db_index=True, verbose_name=_('Start time') default=timezone.now, db_index=True, verbose_name=_("Start time")
) )
last_seen = models.DateTimeField( last_seen = models.DateTimeField(
default=timezone.now, db_index=True, verbose_name=_('Last seen') default=timezone.now, db_index=True, verbose_name=_("Last seen")
) )
# Core request information # Core request information
user_agent = models.TextField(verbose_name=_('User agent')) user_agent = models.TextField(verbose_name=_("User agent"))
browser = models.TextField(verbose_name=_('Browser')) browser = models.TextField(verbose_name=_("Browser"))
device = models.TextField(verbose_name=_('Device')) device = models.TextField(verbose_name=_("Device"))
device_type = models.CharField( device_type = models.CharField(
max_length=7, max_length=7,
choices=[ choices=[
@ -46,23 +45,25 @@ class Session(models.Model):
("OTHER", _("Other")), ("OTHER", _("Other")),
], ],
default="OTHER", default="OTHER",
verbose_name=_('Device type') verbose_name=_("Device type"),
) )
os = models.TextField(verbose_name=_('OS')) os = models.TextField(verbose_name=_("OS"))
ip = models.GenericIPAddressField(db_index=True, null=True, verbose_name=_('IP')) ip = models.GenericIPAddressField(db_index=True, null=True, verbose_name=_("IP"))
# GeoIP data # GeoIP data
asn = models.TextField(blank=True, verbose_name=_('Asn')) asn = models.TextField(blank=True, verbose_name=_("Asn"))
country = models.TextField(blank=True, verbose_name=_('Country')) country = models.TextField(blank=True, verbose_name=_("Country"))
longitude = models.FloatField(null=True, verbose_name=_('Longitude')) longitude = models.FloatField(null=True, verbose_name=_("Longitude"))
latitude = models.FloatField(null=True, verbose_name=_('Latitude')) latitude = models.FloatField(null=True, verbose_name=_("Latitude"))
time_zone = models.TextField(blank=True, verbose_name=_('Time zone')) time_zone = models.TextField(blank=True, verbose_name=_("Time zone"))
is_bounce = models.BooleanField(default=True, db_index=True, verbose_name=_('Is bounce')) is_bounce = models.BooleanField(
default=True, db_index=True, verbose_name=_("Is bounce")
)
class Meta: class Meta:
verbose_name = _('Session') verbose_name = _("Session")
verbose_name_plural = _('Sessions') verbose_name_plural = _("Sessions")
ordering = ["-start_time"] ordering = ["-start_time"]
indexes = [ indexes = [
models.Index(fields=["service", "-start_time"]), models.Index(fields=["service", "-start_time"]),
@ -96,8 +97,7 @@ class Session(models.Model):
class Hit(models.Model): class Hit(models.Model):
session = models.ForeignKey( session = models.ForeignKey(
Session, on_delete=models.CASCADE, db_index=True, Session, on_delete=models.CASCADE, db_index=True, verbose_name=_("Session")
verbose_name=_('Session')
) )
initial = models.BooleanField(default=True, db_index=True) initial = models.BooleanField(default=True, db_index=True)
@ -119,8 +119,8 @@ class Hit(models.Model):
service = models.ForeignKey(Service, on_delete=models.CASCADE, db_index=True) service = models.ForeignKey(Service, on_delete=models.CASCADE, db_index=True)
class Meta: class Meta:
verbose_name = _('Hit') verbose_name = _("Hit")
verbose_name_plural = _('Hits') verbose_name_plural = _("Hits")
ordering = ["-start_time"] ordering = ["-start_time"]
indexes = [ indexes = [
models.Index(fields=["session", "-start_time"]), models.Index(fields=["session", "-start_time"]),
@ -129,7 +129,6 @@ class Hit(models.Model):
models.Index(fields=["session", "referrer"]), models.Index(fields=["session", "referrer"]),
] ]
@property @property
def duration(self): def duration(self):
return self.last_seen - self.start_time return self.last_seen - self.start_time

4
shynet/api/apps.py
View File

@ -2,5 +2,5 @@ from django.apps import AppConfig
class ApiConfig(AppConfig): class ApiConfig(AppConfig):
default_auto_field = 'django.db.models.BigAutoField' default_auto_field = "django.db.models.BigAutoField"
name = 'api' name = "api"

6
shynet/api/mixins.py
View File

@ -6,11 +6,11 @@ from core.models import User
class ApiTokenRequiredMixin: class ApiTokenRequiredMixin:
def _get_user_by_token(self, request): def _get_user_by_token(self, request):
token = request.headers.get('Authorization') token = request.headers.get("Authorization")
if not token or not token.startswith('Token '): if not token or not token.startswith("Token "):
return AnonymousUser() return AnonymousUser()
token = token.split(' ')[1] token = token.split(" ")[1]
user = User.objects.filter(api_token=token).first() user = User.objects.filter(api_token=token).first()
return user if user else AnonymousUser() return user if user else AnonymousUser()

22
shynet/api/views.py
View File

@ -24,7 +24,7 @@ class DashboardApiView(ApiTokenRequiredMixin, DateRangeMixin, View):
Q(owner=request.user) | Q(collaborators__in=[request.user]) Q(owner=request.user) | Q(collaborators__in=[request.user])
).distinct() ).distinct()
uuid = request.GET.get('uuid') uuid = request.GET.get("uuid")
if uuid and is_valid_uuid(uuid): if uuid and is_valid_uuid(uuid):
services = services.filter(uuid=uuid) services = services.filter(uuid=uuid)
@ -32,29 +32,29 @@ class DashboardApiView(ApiTokenRequiredMixin, DateRangeMixin, View):
start = self.get_start_date() start = self.get_start_date()
end = self.get_end_date() end = self.get_end_date()
except ValueError: except ValueError:
return JsonResponse(status=400, data={'error': 'Invalid date format'}) return JsonResponse(status=400, data={"error": "Invalid date format"})
services_data = [ services_data = [
{ {
'name': s.name, "name": s.name,
'uuid': s.uuid, "uuid": s.uuid,
'link': s.link, "link": s.link,
'stats': s.get_core_stats(start, end), "stats": s.get_core_stats(start, end),
} }
for s in services for s in services
] ]
services_data = self._convert_querysets_to_lists(services_data) services_data = self._convert_querysets_to_lists(services_data)
return JsonResponse(data={'services': services_data}) return JsonResponse(data={"services": services_data})
def _convert_querysets_to_lists(self, services_data): def _convert_querysets_to_lists(self, services_data):
for service_data in services_data: for service_data in services_data:
for key, value in service_data['stats'].items(): for key, value in service_data["stats"].items():
if isinstance(value, QuerySet): if isinstance(value, QuerySet):
service_data['stats'][key] = list(value) service_data["stats"][key] = list(value)
for key, value in service_data['stats']['compare'].items(): for key, value in service_data["stats"]["compare"].items():
if isinstance(value, QuerySet): if isinstance(value, QuerySet):
service_data['stats']['compare'][key] = list(value) service_data["stats"]["compare"][key] = list(value)
return services_data return services_data

53
shynet/core/models.py
View File

@ -64,38 +64,51 @@ class Service(models.Model):
SERVICE_STATUSES = [(ACTIVE, _("Active")), (ARCHIVED, _("Archived"))] SERVICE_STATUSES = [(ACTIVE, _("Active")), (ARCHIVED, _("Archived"))]
uuid = models.UUIDField(default=_default_uuid, primary_key=True) uuid = models.UUIDField(default=_default_uuid, primary_key=True)
name = models.TextField(max_length=64, verbose_name=_('Name')) name = models.TextField(max_length=64, verbose_name=_("Name"))
owner = models.ForeignKey( owner = models.ForeignKey(
User, verbose_name=_('Owner'), User,
on_delete=models.CASCADE, related_name="owning_services" verbose_name=_("Owner"),
on_delete=models.CASCADE,
related_name="owning_services",
) )
collaborators = models.ManyToManyField( collaborators = models.ManyToManyField(
User, verbose_name=_('Collaborators'), User,
related_name="collaborating_services", blank=True verbose_name=_("Collaborators"),
related_name="collaborating_services",
blank=True,
) )
created = models.DateTimeField(auto_now_add=True, verbose_name=_('created')) created = models.DateTimeField(auto_now_add=True, verbose_name=_("created"))
link = models.URLField(blank=True, verbose_name=_('link')) link = models.URLField(blank=True, verbose_name=_("link"))
origins = models.TextField(default="*", verbose_name=_('origins')) origins = models.TextField(default="*", verbose_name=_("origins"))
status = models.CharField( status = models.CharField(
max_length=2, choices=SERVICE_STATUSES, default=ACTIVE, db_index=True, max_length=2,
verbose_name=_('status') choices=SERVICE_STATUSES,
default=ACTIVE,
db_index=True,
verbose_name=_("status"),
) )
respect_dnt = models.BooleanField(default=True, verbose_name=_('Respect dnt')) respect_dnt = models.BooleanField(default=True, verbose_name=_("Respect dnt"))
ignore_robots = models.BooleanField(default=False, verbose_name=_('Ignore robots')) ignore_robots = models.BooleanField(default=False, verbose_name=_("Ignore robots"))
collect_ips = models.BooleanField(default=True, verbose_name=_('Collect ips')) collect_ips = models.BooleanField(default=True, verbose_name=_("Collect ips"))
ignored_ips = models.TextField( ignored_ips = models.TextField(
default="", blank=True, validators=[_validate_network_list], default="",
verbose_name=_('Igored ips') blank=True,
validators=[_validate_network_list],
verbose_name=_("Igored ips"),
) )
hide_referrer_regex = models.TextField( hide_referrer_regex = models.TextField(
default="", blank=True, validators=[_validate_regex], default="",
verbose_name=_('Hide referrer regex') blank=True,
validators=[_validate_regex],
verbose_name=_("Hide referrer regex"),
)
script_inject = models.TextField(
default="", blank=True, verbose_name=_("Script inject")
) )
script_inject = models.TextField(default="", blank=True, verbose_name=_('Script inject'))
class Meta: class Meta:
verbose_name = _('Service') verbose_name = _("Service")
verbose_name_plural = _('Services') verbose_name_plural = _("Services")
ordering = ["name", "uuid"] ordering = ["name", "uuid"]
def __str__(self): def __str__(self):

40
shynet/dashboard/forms.py
View File

@ -25,9 +25,15 @@ class ServiceForm(forms.ModelForm):
"name": forms.TextInput(), "name": forms.TextInput(),
"origins": forms.TextInput(), "origins": forms.TextInput(),
"ignored_ips": forms.TextInput(), "ignored_ips": forms.TextInput(),
"respect_dnt": forms.RadioSelect(choices=[(True, _("Yes")), (False, _("No"))]), "respect_dnt": forms.RadioSelect(
"collect_ips": forms.RadioSelect(choices=[(True, _("Yes")), (False, _("No"))]), choices=[(True, _("Yes")), (False, _("No"))]
"ignore_robots": forms.RadioSelect(choices=[(True, _("Yes")), (False, _("No"))]), ),
"collect_ips": forms.RadioSelect(
choices=[(True, _("Yes")), (False, _("No"))]
),
"ignore_robots": forms.RadioSelect(
choices=[(True, _("Yes")), (False, _("No"))]
),
"hide_referrer_regex": forms.TextInput(), "hide_referrer_regex": forms.TextInput(),
"script_inject": forms.Textarea(attrs={"class": "font-mono", "rows": 5}), "script_inject": forms.Textarea(attrs={"class": "font-mono", "rows": 5}),
} }
@ -45,17 +51,29 @@ class ServiceForm(forms.ModelForm):
"origins": _( "origins": _(
"At what origins does the service operate? Use commas to separate multiple values. This sets CORS headers, so use '*' if you're not sure (or don't care)." "At what origins does the service operate? Use commas to separate multiple values. This sets CORS headers, so use '*' if you're not sure (or don't care)."
), ),
"respect_dnt": _("Should visitors who have enabled <a href='https://en.wikipedia.org/wiki/Do_Not_Track'>Do Not Track</a> be excluded from all data?"), "respect_dnt": _(
"ignored_ips": _("A comma-separated list of IP addresses or IP ranges (IPv4 and IPv6) to exclude from tracking (e.g., '192.168.0.2, 127.0.0.1/32')."), "Should visitors who have enabled <a href='https://en.wikipedia.org/wiki/Do_Not_Track'>Do Not Track</a> be excluded from all data?"
"ignore_robots": _("Should sessions generated by bots be excluded from tracking?"), ),
"hide_referrer_regex": _("Any referrers that match this <a href='https://regexr.com/'>RegEx</a> will not be listed in the referrer summary. Sessions will still be tracked normally. No effect if left blank."), "ignored_ips": _(
"script_inject": _("Optional additional JavaScript to inject at the end of the Shynet script. This code will be injected on every page where this service is installed."), "A comma-separated list of IP addresses or IP ranges (IPv4 and IPv6) to exclude from tracking (e.g., '192.168.0.2, 127.0.0.1/32')."
),
"ignore_robots": _(
"Should sessions generated by bots be excluded from tracking?"
),
"hide_referrer_regex": _(
"Any referrers that match this <a href='https://regexr.com/'>RegEx</a> will not be listed in the referrer summary. Sessions will still be tracked normally. No effect if left blank."
),
"script_inject": _(
"Optional additional JavaScript to inject at the end of the Shynet script. This code will be injected on every page where this service is installed."
),
} }
collect_ips = forms.BooleanField( collect_ips = forms.BooleanField(
help_text=_("IP address collection is disabled globally by your administrator.") help_text=_("IP address collection is disabled globally by your administrator.")
if settings.BLOCK_ALL_IPS if settings.BLOCK_ALL_IPS
else _("Should individual IP addresses be collected? IP metadata (location, host, etc) will still be collected."), else _(
"Should individual IP addresses be collected? IP metadata (location, host, etc) will still be collected."
),
widget=forms.RadioSelect(choices=[(True, _("Yes")), (False, _("No"))]), widget=forms.RadioSelect(choices=[(True, _("Yes")), (False, _("No"))]),
initial=False if settings.BLOCK_ALL_IPS else True, initial=False if settings.BLOCK_ALL_IPS else True,
required=False, required=False,
@ -68,7 +86,9 @@ class ServiceForm(forms.ModelForm):
return False if settings.BLOCK_ALL_IPS else collect_ips return False if settings.BLOCK_ALL_IPS else collect_ips
collaborators = forms.CharField( collaborators = forms.CharField(
help_text=_("Which users on this Shynet instance should have read-only access to this service? (Comma separated list of emails.)"), help_text=_(
"Which users on this Shynet instance should have read-only access to this service? (Comma separated list of emails.)"
),
required=False, required=False,
) )

2
shynet/dashboard/views.py
View File

@ -161,4 +161,4 @@ class RefreshApiTokenView(LoginRequiredMixin, View):
def post(self, request): def post(self, request):
request.user.api_token = _default_api_token() request.user.api_token = _default_api_token()
request.user.save() request.user.save()
return redirect('account_change_password') return redirect("account_change_password")