mirrors/gotosocial
1
0
Fork 0
mirror of https://github.com/superseriousbusiness/gotosocial.git synced 2025-10-31 14:32:24 -05:00
Code Issues Projects Releases Packages Wiki Activity Actions

[feature/frogend] (Mastodon) domain block CSV import (#1390)

Browse source 
* checkbox-list styling with taller <p> element

* CSV import/export, UI/UX improvements to import-export interface

* minor styling tweaks

* csv export, clean up export type branching

* abstract domain block entry validation

* foundation for PSL check + suggestions

* Squashed commit of the following:

commit e3655ba4fb
Author: f0x <f0x@cthu.lu>
Date:   Tue Jan 31 15:19:10 2023 +0100

    let debug depend on env (prod/debug) again

commit 79c792b832
Author: f0x <f0x@cthu.lu>
Date:   Tue Jan 31 00:34:01 2023 +0100

    update checklist components

commit 4367960fe4
Author: f0x <f0x@cthu.lu>
Date:   Mon Jan 30 23:46:20 2023 +0100

    checklist performance improvements

commit 204a4c02d1
Author: f0x <f0x@cthu.lu>
Date:   Mon Jan 30 20:05:34 2023 +0100

    checklist field: use reducer for state

* remove debug logging

* show and use domain block suggestion

* restructure import/export buttons

* updating suggestions

* suggestion overview

* restructure check-list behavior, domain import/export
This commit is contained in:
f0x52 2023-02-03 12:07:40 +01:00 • committed by GitHub
commit a59dc855d9
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
17 changed files with 1077 additions and 484 deletions
Download patch file Download diff file Expand all files Collapse all files

50
web/source/settings/lib/domain-block.js Normal file
View file

@ -0,0 +1,50 @@
/*
GoToSocial
Copyright (C) 2021-2023 GoToSocial Authors admin@gotosocial.org
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
"use strict";
const isValidDomain = require("is-valid-domain");
const psl = require("psl");
function isValidDomainBlock(domain) {
return isValidDomain(domain, {
/*
Wildcard prefix *. can be stripped since it's equivalent to not having it,
but wildcard anywhere else in the domain is not handled by the backend so it's invalid.
*/
wildcard: false,
allowUnicode: true
});
}
/*
Still can't think of a better function name for this,
but we're checking a domain against the Public Suffix List <https://publicsuffix.org/>
to see if we should suggest removing subdomain(s) since they're likely owned/ran by the same party
social.example.com -> suggests example.com
*/
function hasBetterScope(domain) {
const lookup = psl.get(domain);
if (lookup && lookup != domain) {
return lookup;
} else {
return false;
}
}
module.exports = { isValidDomainBlock, hasBetterScope };

245
web/source/settings/lib/form/check-list.jsx
View file

@ -20,128 +20,163 @@
const React = require("react");
const syncpipe = require("syncpipe");
const { createSlice } = require("@reduxjs/toolkit");
const { enableMapSet } = require("immer");
function createState(entries, uniqueKey, oldState, defaultValue) {
return syncpipe(entries, [
(_) => _.map((entry) => {
let key = entry[uniqueKey];
return [
key,
{
...entry,
key,
checked: oldState[key]?.checked ?? entry.checked ?? defaultValue
enableMapSet(); // for use in reducers
const { reducer, actions } = createSlice({
name: "checklist",
initialState: {}, // not handled by slice itself
reducers: {
updateAll: (state, { payload: checked }) => {
const selectedEntries = new Set();
return {
entries: syncpipe(state.entries, [
(_) => Object.values(_),
(_) => _.map((entry) => {
if (checked) {
selectedEntries.add(entry.key);
}
return [entry.key, {
...entry,
checked
}];
}),
(_) => Object.fromEntries(_)
]),
selectedEntries
};
},
update: (state, { payload: { key, value } }) => {
if (value.checked !== undefined) {
if (value.checked === true) {
state.selectedEntries.add(key);
} else {
state.selectedEntries.delete(key);
}
];
}),
(_) => Object.fromEntries(_)
]);
}
}
function updateAllState(state, newValue) {
return syncpipe(state, [
(_) => Object.values(_),
(_) => _.map((entry) => [entry.key, {
...entry,
checked: newValue
}]),
(_) => Object.fromEntries(_)
]);
}
state.entries[key] = {
...state.entries[key],
...value
};
},
updateMultiple: (state, { payload }) => {
payload.forEach(([key, value]) => {
if (value.checked !== undefined) {
if (value.checked === true) {
state.selectedEntries.add(key);
} else {
state.selectedEntries.delete(key);
}
}
function updateState(state, key, newValue) {
return {
...state,
[key]: {
...state[key],
...newValue
state.entries[key] = {
...state.entries[key],
...value
};
});
}
}
});
function initialState({ entries, uniqueKey, defaultValue }) {
const selectedEntries = new Set();
return {
entries: syncpipe(entries, [
(_) => _.map((entry) => {
let key = entry[uniqueKey];
let checked = entry.checked ?? defaultValue;
if (checked) {
selectedEntries.add(key);
} else {
selectedEntries.delete(key);
}
return [
key,
{
...entry,
key,
checked
}
];
}),
(_) => Object.fromEntries(_)
]),
selectedEntries
};
}
module.exports = function useCheckListInput({ name }, { entries, uniqueKey = "key", defaultValue = false }) {
const [state, setState] = React.useState({});
const [state, dispatch] = React.useReducer(reducer, null,
() => initialState({ entries, uniqueKey, defaultValue }) // initial state
);
const [someSelected, setSomeSelected] = React.useState(false);
const [toggleAllState, setToggleAllState] = React.useState(0);
const toggleAllRef = React.useRef(null);
React.useEffect(() => {
/*
entries changed, update state,
re-using old state if available for key
*/
setState(createState(entries, uniqueKey, state, defaultValue));
if (toggleAllRef.current != null) {
let some = state.selectedEntries.size > 0;
let all = false;
if (some) {
all = state.selectedEntries.size == Object.values(state.entries).length;
}
toggleAllRef.current.checked = all;
toggleAllRef.current.indeterminate = some && !all;
}
// only needs to update when state.selectedEntries changes, not state.entries
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [state.selectedEntries]);
/* eslint-disable-next-line react-hooks/exhaustive-deps */
}, [entries]);
const reset = React.useCallback(
() => dispatch(actions.updateAll(defaultValue)),
[defaultValue]
);
React.useEffect(() => {
/* Updates (un)check all checkbox, based on shortcode checkboxes
Can be 0 (not checked), 1 (checked) or 2 (indeterminate)
*/
if (toggleAllRef.current == null) {
return;
const onChange = React.useCallback(
(key, value) => dispatch(actions.update({ key, value })),
[]
);
const updateMultiple = React.useCallback(
(entries) => dispatch(actions.updateMultiple(entries)),
[]
);
return React.useMemo(() => {
function toggleAll(e) {
let checked = e.target.checked;
if (e.target.indeterminate) {
checked = false;
}
dispatch(actions.updateAll(checked));
}
let values = Object.values(state);
/* one or more boxes are checked */
let some = values.some((v) => v.checked);
let all = false;
if (some) {
/* there's not at least one unchecked box */
all = !values.some((v) => v.checked == false);
function selectedValues() {
return Array.from((state.selectedEntries)).map((key) => ({
...state.entries[key] // returned as new object, because reducer state is immutable
}));
}
setSomeSelected(some);
if (some && !all) {
setToggleAllState(2);
toggleAllRef.current.indeterminate = true;
} else {
setToggleAllState(all ? 1 : 0);
toggleAllRef.current.indeterminate = false;
}
}, [state, toggleAllRef]);
function toggleAll(e) {
let selectAll = e.target.checked;
if (toggleAllState == 2) { // indeterminate
selectAll = false;
}
setState(updateAllState(state, selectAll));
setToggleAllState(selectAll);
}
function reset() {
setState(updateAllState(state, defaultValue));
}
function selectedValues() {
return syncpipe(state, [
(_) => Object.values(_),
(_) => _.filter((entry) => entry.checked)
]);
}
return Object.assign([
state,
reset,
{ name }
], {
name,
value: state,
onChange: (key, newValue) => setState(updateState(state, key, newValue)),
selectedValues,
reset,
someSelected,
toggleAll: {
ref: toggleAllRef,
value: toggleAllState,
onChange: toggleAll
}
});
return Object.assign([
state,
reset,
{ name }
], {
name,
value: state.entries,
onChange,
selectedValues,
reset,
someSelected: state.someChecked,
updateMultiple,
toggleAll: {
ref: toggleAllRef,
onChange: toggleAll
}
});
}, [state, reset, name, onChange, updateMultiple]);
};

31
web/source/settings/lib/form/text.jsx
View file

@ -20,14 +20,30 @@
const React = require("react");
module.exports = function useTextInput({ name, Name }, { validator, defaultValue = "", dontReset = false } = {}) {
module.exports = function useTextInput({ name, Name }, {
defaultValue = "",
dontReset = false,
validator,
showValidation = true,
initValidation
} = {}) {
const [text, setText] = React.useState(defaultValue);
const [valid, setValid] = React.useState(true);
const textRef = React.useRef(null);
const [validation, setValidation] = React.useState(initValidation ?? "");
const [_isValidating, startValidation] = React.useTransition();
let valid = validation == "";
function onChange(e) {
let input = e.target.value;
setText(input);
if (validator) {
startValidation(() => {
setValidation(validator(input));
});
}
}
function reset() {
@ -38,11 +54,13 @@ module.exports = function useTextInput({ name, Name }, { validator, defaultValue
React.useEffect(() => {
if (validator && textRef.current) {
let res = validator(text);
setValid(res == "");
textRef.current.setCustomValidity(res);
if (showValidation) {
textRef.current.setCustomValidity(validation);
} else {
textRef.current.setCustomValidity("");
}
}
}, [text, textRef, validator]);
}, [validation, validator, showValidation]);
// Array / Object hybrid, for easier access in different contexts
return Object.assign([
@ -62,6 +80,7 @@ module.exports = function useTextInput({ name, Name }, { validator, defaultValue
ref: textRef,
setter: setText,
valid,
validate: () => setValidation(validator(text)),
hasChanged: () => text != defaultValue
});
};

109
web/source/settings/lib/query/admin/import-export.js
View file

@ -19,8 +19,11 @@
"use strict";
const Promise = require("bluebird");
const isValidDomain = require("is-valid-domain");
const fileDownload = require("js-file-download");
const csv = require("papaparse");
const { nanoid } = require("nanoid");
const { isValidDomainBlock, hasBetterScope } = require("../../domain-block");
const {
replaceCacheOnMutation,
@ -31,6 +34,23 @@ const {
function parseDomainList(list) {
if (list[0] == "[") {
return JSON.parse(list);
} else if (list.startsWith("#domain")) { // Mastodon CSV
const { data, errors } = csv.parse(list, {
header: true,
transformHeader: (header) => header.slice(1), // removes starting '#'
skipEmptyLines: true,
dynamicTyping: true
});
if (errors.length > 0) {
let error = "";
errors.forEach((err) => {
error += `${err.message} (line ${err.row})`;
});
throw error;
}
return data;
} else {
return list.split("\n").map((line) => {
let domain = line.trim();
@ -51,7 +71,15 @@ function parseDomainList(list) {
function validateDomainList(list) {
list.forEach((entry) => {
entry.valid = (entry.valid !== false) && isValidDomain(entry.domain, { wildcard: true, allowUnicode: true });
if (entry.domain.startsWith("*.")) {
// domain block always includes all subdomains, wildcard is meaningless here
entry.domain = entry.domain.slice(2);
}
entry.valid = (entry.valid !== false) && isValidDomainBlock(entry.domain);
if (entry.valid) {
entry.suggest = hasBetterScope(entry.domain);
}
entry.checked = entry.valid;
});
@ -83,6 +111,9 @@ module.exports = (build) => ({
}).then((deduped) => {
return validateDomainList(deduped);
}).then((data) => {
data.forEach((entry) => {
entry.key = nanoid(); // unique id that stays stable even if domain gets modified by user
});
return { data };
}).catch((e) => {
return { error: e.toString() };
@ -91,27 +122,53 @@ module.exports = (build) => ({
}),
exportDomainList: build.mutation({
queryFn: (formData, api, _extraOpts, baseQuery) => {
let process;
if (formData.exportType == "json") {
process = {
transformEntry: (entry) => ({
domain: entry.domain,
public_comment: entry.public_comment,
obfuscate: entry.obfuscate
}),
stringify: (list) => JSON.stringify(list),
extension: ".json",
mime: "application/json"
};
} else if (formData.exportType == "csv") {
process = {
transformEntry: (entry) => [
entry.domain,
"suspend", // severity
false, // reject_media
false, // reject_reports
entry.public_comment,
entry.obfuscate ?? false
],
stringify: (list) => csv.unparse({
fields: "#domain,#severity,#reject_media,#reject_reports,#public_comment,#obfuscate".split(","),
data: list
}),
extension: ".csv",
mime: "text/csv"
};
} else {
process = {
transformEntry: (entry) => entry.domain,
stringify: (list) => list.join("\n"),
extension: ".txt",
mime: "text/plain"
};
}
return Promise.try(() => {
return baseQuery({
url: `/api/v1/admin/domain_blocks`
});
}).then(unwrapRes).then((blockedInstances) => {
return blockedInstances.map((entry) => {
if (formData.exportType == "json") {
return {
domain: entry.domain,
public_comment: entry.public_comment
};
} else {
return entry.domain;
}
});
return blockedInstances.map(process.transformEntry);
}).then((exportList) => {
if (formData.exportType == "json") {
return JSON.stringify(exportList);
} else {
return exportList.join("\n");
}
return process.stringify(exportList);
}).then((exportAsString) => {
if (formData.action == "export") {
return {
@ -120,7 +177,6 @@ module.exports = (build) => ({
} else if (formData.action == "export-file") {
let domain = new URL(api.getState().oauth.instance).host;
let date = new Date();
let mime;
let filename = [
domain,
@ -130,15 +186,11 @@ module.exports = (build) => ({
date.getDate().toString().padStart(2, "0"),
].join("-");
if (formData.exportType == "json") {
filename += ".json";
mime = "application/json";
} else {
filename += ".txt";
mime = "text/plain";
}
fileDownload(exportAsString, filename, mime);
fileDownload(
exportAsString,
filename + process.extension,
process.mime
);
}
return { data: null };
}).catch((e) => {
@ -171,6 +223,7 @@ module.exports = (build) => ({
})
});
const internalKeys = new Set("key,suggest,valid,checked".split(","));
function entryProcessor(formData) {
let funcs = [];
@ -204,7 +257,7 @@ function entryProcessor(formData) {
entry.obfuscate = formData.obfuscate;
Object.entries(entry).forEach(([key, val]) => {
if (val == undefined) {
if (internalKeys.has(key) || val == undefined) {
delete entry[key];
}
});