Compare commits

...

9 Commits

Author SHA1 Message Date
23a466cd72
small fixes 2025-05-07 03:49:25 +03:00
305cb6eca6
small refactoring 2025-05-06 23:40:06 +03:00
07f32f8e95
small refactoring 2025-05-06 23:35:28 +03:00
0e32cc3f17
extract from attrubite 2025-05-06 20:47:10 +03:00
f220f9d9d7
small refactoring 2025-05-06 20:31:02 +03:00
4302176348
extract from field 2025-05-06 19:03:58 +03:00
01c28aead2
extract from field 2025-05-06 19:01:18 +03:00
da86d3b5d2
frontend add new field 2025-05-06 16:51:07 +03:00
ae2cdb4f14
frontend fields refactoring 2025-05-06 16:49:16 +03:00
18 changed files with 478 additions and 201 deletions

View File

@ -0,0 +1,5 @@
export type EnumValue = {
label: string
value: number
}
export type Enum = EnumValue[]

View File

@ -1,20 +1,10 @@
<script setup lang="ts">
import Field from "@/components/Field.vue";
import {type Field as FieldSpec} from "@/urlmaker/specs";
import {validateOr, validatePreset, validateUrl} from "@/urlmaker/validators.ts";
import TextField from "@/components/inputs/TextField.vue";
import Btn from "@/components/Btn.vue";
import {onMounted, onUnmounted, ref, watch} from "vue";
import Modal from "@/components/Modal.vue";
const field: FieldSpec = {
name: '',
input_type: 'url',
label: 'URL of feed or preset',
default: '',
required: true,
validate: validateOr(validateUrl, validatePreset),
}
import {validatePreset, validateUrl} from "@/urlmaker/validators.ts";
const visible = defineModel('visible', {
type: Boolean,
@ -33,11 +23,10 @@ watch(visible, () => {
const valid = ref(false);
watch(url, (value) => {
valid.value = field.validate(value).ok;
valid.value = validateUrl(value) || validatePreset(value);
});
const accept = () => {
valid.value = field.validate(url.value).ok;
if (valid.value) {
emit('update:modelValue', url.value);
emit('update:visible', false);
@ -59,7 +48,13 @@ onUnmounted(() => {
<template>
<Modal v-model="visible">
<Field :field="field" v-model="url" :focused="true"/>
<TextField
name="url"
input_type="url"
label="URL of feed or preset"
v-model="url"
:focused="true"
/>
<Btn :active="valid" @click="accept">Edit</Btn>
</Modal>
</template>

View File

@ -1,22 +1,51 @@
<script setup lang="ts">
import {fields, type Specs} from '@/urlmaker/specs.ts';
import Field from "@/components/Field.vue";
import {fields, InputType, type SpecField} from '@/urlmaker/specs.ts';
import TextField from "@/components/inputs/TextField.vue";
import RadioButtons from "@/components/inputs/RadioButtons.vue";
import {useWizardStore} from "@/stores/wizard.ts";
const store = useWizardStore();
const groups: SpecField[][] = [];
for (const field of fields) {
if(groups.length === 0 || groups[groups.length - 1][0].group != field.group) {
groups.push([field]);
} else {
groups[groups.length - 1].push(field);
}
}
</script>
<template>
<div>
<Field v-for="field in fields"
:field="field"
<div class="group" v-for="group in Object.values(groups)">
<template v-for="field in group">
<TextField
v-if="field.input_type === InputType.Url || field.input_type === InputType.Text"
v-show="!field.show_if || field.show_if(store.specs)"
:name="field.name"
:label="field.label"
:input_type="field.input_type"
:model-value="store.specs[field.name]"
@update:model-value="event => store.updateSpec(field.name, event)"
></Field>
></TextField>
<RadioButtons
v-if="field.input_type === InputType.Radio"
v-show="!field.show_if || field.show_if(store.specs)"
:name="field.name"
:label="field.label"
:values="field.enum!"
:model-value="store.specs[field.name]"
@update:model-value="event => store.updateSpec(field.name, event)"
></RadioButtons>
</template>
</div>
</div>
</template>
<style scoped lang="scss">
div.group {
margin: 20px 0;
}
</style>

View File

@ -0,0 +1,51 @@
<script setup lang="ts">
import {getCurrentInstance} from "vue";
import type {Enum} from "@/common/enum.ts";
const {name, label, values} = defineProps<{
name: string
label: string,
values: Enum,
}>();
const componentId = 'field' + getCurrentInstance()?.uid;
const model = defineModel();
</script>
<template>
<div class="field">
<span class="field-label"><label>{{ label }}</label></span>
<template class="value" v-for="enumValue in values">
<input
type="radio"
:name="name"
:value="enumValue.value"
:id="`${componentId}_${enumValue.value}`"
v-model="model"
/>
<label class="radio-label" :for="`${componentId}_${enumValue.value}`">{{ enumValue.label }}</label>
</template>
</div>
</template>
<style scoped lang="scss">
div.field {
margin: 0 0 8px 0;
}
.field-label {
font-size: 0.9em;
margin-right: 8px;
}
.radio-label {
font-size: 0.9em;
}
input, .radio-label, .field-label {
vertical-align: middle;
}
input {
margin-top: 0;
}
</style>

View File

@ -1,9 +1,10 @@
<script setup lang="ts">
import type {Field} from "@/urlmaker/specs.ts";
import {getCurrentInstance, onMounted, useTemplateRef} from "vue";
const {field, focused} = defineProps<{
field: Field,
const {name, label, input_type, focused} = defineProps<{
name: string
label: string,
input_type: 'text' | 'url',
focused?: boolean,
}>();
const id = 'field' + getCurrentInstance()?.uid;
@ -18,9 +19,9 @@ onMounted(() => {
<template>
<div class="field">
<div class="label"><label :for="id">{{ field.label }}</label></div>
<div class="label"><label :for="id">{{ label }}</label></div>
<div class="input">
<input :type="field.input_type" :name="field.name" :id="id" v-model="model" ref="field"/>
<input :type="input_type" :name="name" :id="id" v-model="model" ref="field"/>
</div>
</div>
</template>

View File

@ -19,8 +19,8 @@ watch(existingLink, async (value) => {
if(!value) return;
existingLink.value = "";
try {
if(validateUrl(value).ok) store.updateSpecs(await decodeUrl(value));
else if (validatePreset(value).ok) store.updateSpecs(await decodePreset(value));
if(validateUrl(value)) store.updateSpecs(await decodeUrl(value));
else if (validatePreset(value)) store.updateSpecs(await decodePreset(value));
} catch (e) {
console.log(e);
alert(`Decoding error: ${e}`);
@ -60,7 +60,6 @@ function screenshot() {
<template>
<div class="wrapper">
<SpecsForm class="specs-form"></SpecsForm>
<!-- <Btn :active="store.formValid" @click="generateLink">Generate link</Btn>-->
<Btn :active="store.formValid" @click="screenshot">Screenshot</Btn>
<Btn @click="editModalVisible = true">Edit existing task / import preset</Btn>
<Btn @click="store.reset">Reset Form</Btn>

View File

@ -1,5 +1,5 @@
import {defineStore} from "pinia";
import {emptySpecs, type Field, type FieldNames, fields, type Specs} from "@/urlmaker/specs.ts";
import {defaultSpecs, type SpecField, fields, type Specs, type SpecValue} from "@/urlmaker/specs.ts";
import {computed, reactive} from "vue";
import {debounce} from "es-toolkit";
@ -8,13 +8,13 @@ const LOCAL_STORAGE_KEY = 'rssalchemy_store_wizard';
export const useWizardStore = defineStore('wizard', () => {
const locStorageContent = localStorage.getItem(LOCAL_STORAGE_KEY);
const initialSpecs = locStorageContent ? JSON.parse(locStorageContent) as Specs : emptySpecs;
const initialSpecs = locStorageContent ? JSON.parse(locStorageContent) as Specs : defaultSpecs;
const specs = reactive(Object.assign({}, initialSpecs));
const formValid = computed(() => {
return fields.every(field => (
specs[field.name].length === 0 && !(field as Field).required || field.validate(specs[field.name]).ok
!specs[field.name] && !(field as SpecField).required || field.validate(specs[field.name]!)
));
});
@ -22,8 +22,8 @@ export const useWizardStore = defineStore('wizard', () => {
localStorage.setItem(LOCAL_STORAGE_KEY, JSON.stringify(specs));
}, 100);
function updateSpec(fieldName: FieldNames, newValue: string) {
specs[fieldName] = newValue;
function updateSpec(fieldName: keyof Specs, newValue: SpecValue) {
(specs as Record<keyof Specs, SpecValue>)[fieldName] = newValue;
updateLocalStorage();
}
function updateSpecs(newValue: Specs) {
@ -31,7 +31,7 @@ export const useWizardStore = defineStore('wizard', () => {
updateLocalStorage();
}
function reset() {
Object.assign(specs, emptySpecs);
Object.assign(specs, defaultSpecs);
updateLocalStorage();
}

View File

@ -1,6 +1,6 @@
import type {Specs} from "@/urlmaker/specs.ts";
import {type Specs} from "@/urlmaker/specs.ts";
import {b64decode, b64encode, compress, decompress, decompressString} from "@/urlmaker/utils.ts";
import {rssalchemy as pb} from '@/urlmaker/proto/specs.ts';
import {rssalchemy, rssalchemy as pb} from '@/urlmaker/proto/specs.ts';
const apiBase = import.meta.env.VITE_API_BASE || document.location.origin;
const renderEndpoint = '/api/v1/render/'; // trailing slash
@ -57,7 +57,7 @@ export async function encodePreset(specs: Specs): Promise<string> {
}
export async function encodeSpecsPart(specs: Specs): Promise<string> {
const pbSpecs = pb.Specs.fromObject(specs);
const pbSpecs = pb.Specs.fromObject(specs as ReturnType<rssalchemy.Specs['toObject']>);
let data = pbSpecs.serializeBinary();
data = await compress(data);
const encodedData = b64encode(data);

View File

@ -5,6 +5,10 @@
* git: https://github.com/thesayyn/protoc-gen-ts */
import * as pb_1 from "google-protobuf";
export namespace rssalchemy {
export enum ExtractFrom {
InnerText = 0,
Attribute = 1
}
export class Specs extends pb_1.Message {
#one_of_decls: number[][] = [];
constructor(data?: any[] | {
@ -15,6 +19,8 @@ export namespace rssalchemy {
selector_description?: string;
selector_author?: string;
selector_created?: string;
created_extract_from?: ExtractFrom;
created_attribute_name?: string;
selector_content?: string;
selector_enclosure?: string;
cache_lifetime?: string;
@ -43,6 +49,12 @@ export namespace rssalchemy {
if ("selector_created" in data && data.selector_created != undefined) {
this.selector_created = data.selector_created;
}
if ("created_extract_from" in data && data.created_extract_from != undefined) {
this.created_extract_from = data.created_extract_from;
}
if ("created_attribute_name" in data && data.created_attribute_name != undefined) {
this.created_attribute_name = data.created_attribute_name;
}
if ("selector_content" in data && data.selector_content != undefined) {
this.selector_content = data.selector_content;
}
@ -96,6 +108,18 @@ export namespace rssalchemy {
set selector_created(value: string) {
pb_1.Message.setField(this, 7, value);
}
get created_extract_from() {
return pb_1.Message.getFieldWithDefault(this, 11, ExtractFrom.InnerText) as ExtractFrom;
}
set created_extract_from(value: ExtractFrom) {
pb_1.Message.setField(this, 11, value);
}
get created_attribute_name() {
return pb_1.Message.getFieldWithDefault(this, 12, "") as string;
}
set created_attribute_name(value: string) {
pb_1.Message.setField(this, 12, value);
}
get selector_content() {
return pb_1.Message.getFieldWithDefault(this, 8, "") as string;
}
@ -122,6 +146,8 @@ export namespace rssalchemy {
selector_description?: string;
selector_author?: string;
selector_created?: string;
created_extract_from?: ExtractFrom;
created_attribute_name?: string;
selector_content?: string;
selector_enclosure?: string;
cache_lifetime?: string;
@ -148,6 +174,12 @@ export namespace rssalchemy {
if (data.selector_created != null) {
message.selector_created = data.selector_created;
}
if (data.created_extract_from != null) {
message.created_extract_from = data.created_extract_from;
}
if (data.created_attribute_name != null) {
message.created_attribute_name = data.created_attribute_name;
}
if (data.selector_content != null) {
message.selector_content = data.selector_content;
}
@ -168,6 +200,8 @@ export namespace rssalchemy {
selector_description?: string;
selector_author?: string;
selector_created?: string;
created_extract_from?: ExtractFrom;
created_attribute_name?: string;
selector_content?: string;
selector_enclosure?: string;
cache_lifetime?: string;
@ -193,6 +227,12 @@ export namespace rssalchemy {
if (this.selector_created != null) {
data.selector_created = this.selector_created;
}
if (this.created_extract_from != null) {
data.created_extract_from = this.created_extract_from;
}
if (this.created_attribute_name != null) {
data.created_attribute_name = this.created_attribute_name;
}
if (this.selector_content != null) {
data.selector_content = this.selector_content;
}
@ -222,6 +262,10 @@ export namespace rssalchemy {
writer.writeString(6, this.selector_author);
if (this.selector_created.length)
writer.writeString(7, this.selector_created);
if (this.created_extract_from != ExtractFrom.InnerText)
writer.writeEnum(11, this.created_extract_from);
if (this.created_attribute_name.length)
writer.writeString(12, this.created_attribute_name);
if (this.selector_content.length)
writer.writeString(8, this.selector_content);
if (this.selector_enclosure.length)
@ -258,6 +302,12 @@ export namespace rssalchemy {
case 7:
message.selector_created = reader.readString();
break;
case 11:
message.created_extract_from = reader.readEnum();
break;
case 12:
message.created_attribute_name = reader.readString();
break;
case 8:
message.selector_content = reader.readString();
break;

View File

@ -1,98 +1,132 @@
import {
validateAttribute,
validateDuration,
validateSelector,
validateUrl,
type validator
} from "@/urlmaker/validators.ts";
import {rssalchemy} from "@/urlmaker/proto/specs.ts";
import type {Enum} from "@/common/enum.ts";
export interface Field {
name: string
input_type: string
label: string
default: string
validate: validator
required?: boolean
export const defaultSpecs = {
url: '',
selector_post: '',
selector_title: '',
selector_link: '',
selector_description: '',
selector_author: '',
selector_content: '',
selector_enclosure: '',
selector_created: '',
created_extract_from: rssalchemy.ExtractFrom.InnerText,
created_attribute_name: '',
cache_lifetime: '10m'
};
export type SpecValue = string | number;
export type Specs = typeof defaultSpecs;
export enum InputType {
Url = 'url',
Text = 'text',
Radio = 'radio'
}
export const fields = [
export interface SpecField {
name: keyof Specs
input_type: InputType
enum?: Enum,
label: string
validate: validator
required?: boolean
group?: string
show_if?: (specs: Specs) => boolean
}
export const fields: SpecField[] = [
{
name: 'url',
input_type: 'url',
input_type: InputType.Url,
label: 'URL of page for converting',
default: '',
validate: validateUrl,
required: true,
},
{
name: 'selector_post',
input_type: 'text',
input_type: InputType.Text,
label: 'CSS Selector for post',
default: '',
validate: validateSelector,
},
{
name: 'selector_title',
input_type: 'text',
input_type: InputType.Text,
label: 'CSS Selector for title',
default: '',
validate: validateSelector,
},
{
name: 'selector_link',
input_type: 'text',
input_type: InputType.Text,
label: 'CSS Selector for link',
default: '',
validate: validateSelector,
},
{
name: 'selector_description',
input_type: 'text',
input_type: InputType.Text,
label: 'CSS Selector for description',
default: '',
validate: validateSelector,
},
{
name: 'selector_author',
input_type: 'text',
input_type: InputType.Text,
label: 'CSS Selector for author',
default: '',
validate: validateSelector,
},
{
name: 'selector_created',
input_type: 'text',
input_type: InputType.Text,
label: 'CSS Selector for created date',
default: '',
validate: validateSelector,
group: 'created',
},
{
name: 'created_extract_from',
input_type: InputType.Radio,
enum: [
{label: 'Inner Text', value: rssalchemy.ExtractFrom.InnerText},
{label: 'Attribute', value: rssalchemy.ExtractFrom.Attribute},
],
label: 'Extract from',
validate: value => Object.values(rssalchemy.ExtractFrom).includes(value),
group: 'created',
show_if: specs => !!specs.selector_created,
},
{
name: 'created_attribute_name',
input_type: InputType.Text,
label: 'Attribute name',
validate: validateAttribute,
show_if: specs =>
!!specs.selector_created && specs.created_extract_from === rssalchemy.ExtractFrom.Attribute,
group: 'created',
},
{
name: 'selector_content',
input_type: 'text',
input_type: InputType.Text,
label: 'CSS Selector for content',
default: '',
validate: validateSelector,
},
{
name: 'selector_enclosure',
input_type: 'text',
input_type: InputType.Text,
label: 'CSS Selector for enclosure (e.g. image url)',
default: '',
validate: validateSelector,
},
{
name: 'cache_lifetime',
input_type: 'text',
input_type: InputType.Text,
label: 'Cache lifetime (format examples: 10s, 1m, 2h)',
default: '10m',
validate: validateDuration,
},
] as const satisfies Field[];
export type FieldNames = (typeof fields)[number]['name'];
export type Specs = {[k in FieldNames]: string};
export const emptySpecs = fields.reduce((o, f) => {
o[f.name] = f.default;
return o
}, {} as Specs);
];

View File

@ -1,54 +1,35 @@
import {presetPrefix} from "@/urlmaker/index.ts";
import type {SpecValue} from "@/urlmaker/specs.ts";
type validResult = { ok: boolean, error?: string };
export type validator = (v: string) => validResult
export type validator = (v: SpecValue) => boolean;
export function validateUrl(s: string): validResult {
export function validateUrl(s: SpecValue): boolean {
let url;
try {
url = new URL(s);
return {
ok: url.protocol === "http:" || url.protocol === "https:",
error: 'Invalid URL protocol',
};
url = new URL(s as string);
return url.protocol === "http:" || url.protocol === "https:"
} catch {
return {ok: false, error: 'Invalid URL'};
return false;
}
}
export function validatePreset(s: string): validResult {
if(!s.startsWith(presetPrefix)) {
return {
ok: false,
error: 'Not a preset'
}
}
return {ok: true}
export function validatePreset(s: SpecValue): boolean {
return (s as string).startsWith(presetPrefix);
}
export function validateOr(...validators: validator[]): validator {
return function(s: string): validResult {
return validators.reduce<validResult>((res, v) => {
let r = v(s);
if(r.ok) res.ok = true;
else res.error += r.error + '; ';
return res;
}, {ok: false, error: ''});
}
}
export function validateSelector(s: string): validResult {
export function validateSelector(s: SpecValue): boolean {
try {
document.createDocumentFragment().querySelector(s);
return {ok: true}
document.createDocumentFragment().querySelector(s as string);
return true;
} catch {
return {ok: false, error: 'Invalid selector'};
return false;
}
}
export function validateDuration(s: string): validResult {
return {
ok: /^\d+[smh]$/.test(s),
error: 'Duration must be number and unit (s/m/h), example: 5s = 5 seconds'
}
export function validateAttribute(s: SpecValue): boolean {
return /([^\t\n\f \/>"'=]+)/.test(s as string);
}
export function validateDuration(s: SpecValue): boolean {
return /^\d+[smh]$/.test(s as string);
}

4
go.mod
View File

@ -5,6 +5,7 @@ go 1.23.2
toolchain go1.24.0
require (
github.com/AdguardTeam/urlfilter v0.20.0
github.com/ericchiang/css v1.4.0
github.com/felixge/fgprof v0.9.5
github.com/go-playground/validator/v10 v10.23.0
@ -27,7 +28,6 @@ require (
require (
github.com/AdguardTeam/golibs v0.29.0 // indirect
github.com/AdguardTeam/urlfilter v0.20.0 // indirect
github.com/BurntSushi/toml v1.2.1 // indirect
github.com/alessandro-c/gomemcached-lock v1.0.0 // indirect
github.com/armon/go-metrics v0.4.1 // indirect
@ -112,4 +112,4 @@ require (
olympos.io/encoding/edn v0.0.0-20201019073823-d3554ca0b0a3 // indirect
)
replace github.com/ericchiang/css => github.com/egor3f/css v0.0.0-20250115151140-52c8c51084e5
replace github.com/ericchiang/css => github.com/egor3f/css v0.0.0-20250507004805-bfefe22b74a4

23
go.sum
View File

@ -93,8 +93,8 @@ github.com/deckarep/golang-set/v2 v2.7.0 h1:gIloKvD7yH2oip4VLhsv3JyLLFnC0Y2mlusg
github.com/deckarep/golang-set/v2 v2.7.0/go.mod h1:VAky9rY/yGXJOLEDv3OMci+7wtDpOF4IN+y82NBOac4=
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
github.com/egor3f/css v0.0.0-20250115151140-52c8c51084e5 h1:rqpFTlOasDC5OXOf8NA+XEdjPClBnPGxsQ484OXx6l4=
github.com/egor3f/css v0.0.0-20250115151140-52c8c51084e5/go.mod h1:sVSdL+MFR9Q4cKJMQzpIkHIDOLiK+7Wmjjhq7D+MubA=
github.com/egor3f/css v0.0.0-20250507004805-bfefe22b74a4 h1:hDS4GEOnI8sYW2BqAzN9EA9Ks/3yOQGhOoO4/sjpDzw=
github.com/egor3f/css v0.0.0-20250507004805-bfefe22b74a4/go.mod h1:sVSdL+MFR9Q4cKJMQzpIkHIDOLiK+7Wmjjhq7D+MubA=
github.com/elliotchance/pie/v2 v2.7.0 h1:FqoIKg4uj0G/CrLGuMS9ejnFKa92lxE1dEgBD3pShXg=
github.com/elliotchance/pie/v2 v2.7.0/go.mod h1:18t0dgGFH006g4eVdDtWfgFZPQEgl10IoEO8YWEq3Og=
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
@ -115,6 +115,8 @@ github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2
github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY=
github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s=
github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA=
@ -264,6 +266,8 @@ github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ=
github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI=
github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4=
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I=
github.com/lyft/protoc-gen-star/v2 v2.0.3/go.mod h1:amey7yeodaJhXSbf/TlLvWiqQfLOSpEk//mLlc+axEk=
github.com/magefile/mage v1.14.0 h1:6QDX3g6z1YvJ4olPhT1wksUcSa/V0a1B+pJb73fBjyo=
github.com/magefile/mage v1.14.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A=
@ -288,7 +292,6 @@ github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5
github.com/mennanov/limiters v1.11.0 h1:nTv4uSl3EAc+2fO4B3LXfxkcQe58LeKF9vmwIa3m6Lo=
github.com/mennanov/limiters v1.11.0/go.mod h1:NFf49GLfiywZ4DFkqK9Ne7e+Ckwl1q0eSU+ALwSAxBk=
github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso=
github.com/miekg/dns v1.1.41 h1:WMszZWJG0XmzbK9FEmzH2TVcqYzFesusSIB41b8KHxY=
github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI=
github.com/miekg/dns v1.1.61 h1:nLxbwF3XxhwVSm8g9Dghm9MHPaUZuqhPiGL+675ZmEs=
github.com/miekg/dns v1.1.61/go.mod h1:mnAarhS3nWaW+NVP2wTkYVIZyHNJ098SJZUki3eykwQ=
@ -337,6 +340,8 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRI
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
github.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSgv7Sy7s/s=
github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw=
github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE=
github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU=
@ -358,6 +363,10 @@ github.com/samuel/go-zookeeper v0.0.0-20201211165307-7117e9ea2414 h1:AJNDS0kP60X
github.com/samuel/go-zookeeper v0.0.0-20201211165307-7117e9ea2414/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E=
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I=
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
github.com/shirou/gopsutil/v3 v3.24.5 h1:i0t8kL+kQTvpAYToeuiVk3TgDeKOFioZO3Ztz/iZ9pI=
github.com/shirou/gopsutil/v3 v3.24.5/go.mod h1:bsoOS1aStSs9ErQ1WWfxllSeS1K5D+U30r2NfcubMVk=
github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM=
github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ=
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
github.com/spf13/afero v1.3.3/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4=
@ -388,6 +397,10 @@ github.com/tetratelabs/wazero v1.2.1 h1:J4X2hrGzJvt+wqltuvcSjHQ7ujQxA9gb6PeMs4ql
github.com/tetratelabs/wazero v1.2.1/go.mod h1:wYx2gNRg8/WihJfSDxA1TIL8H+GkfLYm+bIfbblu9VQ=
github.com/thanhpk/randstr v1.0.4 h1:IN78qu/bR+My+gHCvMEXhR/i5oriVHcTB/BJJIRTsNo=
github.com/thanhpk/randstr v1.0.4/go.mod h1:M/H2P1eNLZzlDwAzpkkkUvoyNNMbzRGhESZuEQk3r0U=
github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU=
github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI=
github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk=
github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY=
github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM=
github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
@ -400,6 +413,8 @@ github.com/wasilibs/nottinygc v0.4.0/go.mod h1:oDcIotskuYNMpqMF23l7Z8uzD4TC0WXHK
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0=
github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
go.etcd.io/etcd/api/v3 v3.5.17 h1:cQB8eb8bxwuxOilBpMJAEo8fAONyrdXTHUNcMd8yT1w=
go.etcd.io/etcd/api/v3 v3.5.17/go.mod h1:d1hvkRuXkts6PmaYk2Vrgqbv7H4ADfAKhyJqHNLJCB4=
go.etcd.io/etcd/client/pkg/v3 v3.5.17 h1:XxnDXAWq2pnxqx76ljWwiQ9jylbpC4rvkAeRVOUKKVw=
@ -424,8 +439,6 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y
golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
golang.org/x/crypto v0.32.0 h1:euUpcYgM8WcP71gNpTqQCn6rC2t6ULUPiOzfWaXVVfc=
golang.org/x/crypto v0.32.0/go.mod h1:ZnnJkOaASj8g0AjIduWNlq2NRxL0PlBrbKVyZ6V/Ugc=
golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63 h1:m64FZMko/V45gv0bNmrNYoDEq8U5YUhetc9cBWKS1TQ=
golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63/go.mod h1:0v4NqG35kSWCMzLaMeX+IQrlSnVE/bqGSyC2cz/9Le8=
golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 h1:e66Fs6Z+fZTbFBAxKfP3PALWBtpfqks2bwGcexMxgtk=
golang.org/x/exp v0.0.0-20240909161429-701f63a606c0/go.mod h1:2TbTHSBQa924w8M6Xs1QcRcFwyucIwBGpK1p2f1YFFY=
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=

View File

@ -75,6 +75,14 @@ func (h *Handler) handleRender(c echo.Context) error {
return echo.NewHTTPError(400, fmt.Errorf("decode specs: %w", err))
}
extractFrom, ok := map[pb.ExtractFrom]models.ExtractFrom{
pb.ExtractFrom_InnerText: models.ExtractFrom_InnerText,
pb.ExtractFrom_Attribute: models.ExtractFrom_Attribute,
}[specs.CreatedExtractFrom]
if !ok {
return echo.NewHTTPError(400, "invalid extract from")
}
task := models.Task{
TaskType: models.TaskTypeExtract,
URL: specs.Url,
@ -84,6 +92,8 @@ func (h *Handler) handleRender(c echo.Context) error {
SelectorDescription: specs.SelectorDescription,
SelectorAuthor: specs.SelectorAuthor,
SelectorCreated: specs.SelectorCreated,
CreatedExtractFrom: extractFrom,
CreatedAttributeName: specs.CreatedAttributeName,
SelectorContent: specs.SelectorContent,
SelectorEnclosure: specs.SelectorEnclosure,
Headers: extractHeaders(c),

View File

@ -22,6 +22,52 @@ const (
_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
)
type ExtractFrom int32
const (
ExtractFrom_InnerText ExtractFrom = 0
ExtractFrom_Attribute ExtractFrom = 1
)
// Enum value maps for ExtractFrom.
var (
ExtractFrom_name = map[int32]string{
0: "InnerText",
1: "Attribute",
}
ExtractFrom_value = map[string]int32{
"InnerText": 0,
"Attribute": 1,
}
)
func (x ExtractFrom) Enum() *ExtractFrom {
p := new(ExtractFrom)
*p = x
return p
}
func (x ExtractFrom) String() string {
return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x))
}
func (ExtractFrom) Descriptor() protoreflect.EnumDescriptor {
return file_proto_specs_proto_enumTypes[0].Descriptor()
}
func (ExtractFrom) Type() protoreflect.EnumType {
return &file_proto_specs_proto_enumTypes[0]
}
func (x ExtractFrom) Number() protoreflect.EnumNumber {
return protoreflect.EnumNumber(x)
}
// Deprecated: Use ExtractFrom.Descriptor instead.
func (ExtractFrom) EnumDescriptor() ([]byte, []int) {
return file_proto_specs_proto_rawDescGZIP(), []int{0}
}
type Specs struct {
state protoimpl.MessageState `protogen:"open.v1"`
Url string `protobuf:"bytes,1,opt,name=url,proto3" json:"url" validate:"url"`
@ -29,8 +75,10 @@ type Specs struct {
SelectorTitle string `protobuf:"bytes,3,opt,name=selector_title,json=selectorTitle,proto3" json:"selector_title" validate:"selector"`
SelectorLink string `protobuf:"bytes,4,opt,name=selector_link,json=selectorLink,proto3" json:"selector_link" validate:"selector"`
SelectorDescription string `protobuf:"bytes,5,opt,name=selector_description,json=selectorDescription,proto3" json:"selector_description" validate:"omitempty,selector"`
SelectorAuthor string `protobuf:"bytes,6,opt,name=selector_author,json=selectorAuthor,proto3" json:"selector_author" validate:"selector"`
SelectorAuthor string `protobuf:"bytes,6,opt,name=selector_author,json=selectorAuthor,proto3" json:"selector_author" validate:"omitempty,selector"`
SelectorCreated string `protobuf:"bytes,7,opt,name=selector_created,json=selectorCreated,proto3" json:"selector_created" validate:"selector"`
CreatedExtractFrom ExtractFrom `protobuf:"varint,11,opt,name=created_extract_from,json=createdExtractFrom,proto3,enum=rssalchemy.ExtractFrom" json:"created_extract_from"`
CreatedAttributeName string `protobuf:"bytes,12,opt,name=created_attribute_name,json=createdAttributeName,proto3" json:"created_attribute_name"`
SelectorContent string `protobuf:"bytes,8,opt,name=selector_content,json=selectorContent,proto3" json:"selector_content" validate:"omitempty,selector"`
SelectorEnclosure string `protobuf:"bytes,9,opt,name=selector_enclosure,json=selectorEnclosure,proto3" json:"selector_enclosure" validate:"selector"`
CacheLifetime string `protobuf:"bytes,10,opt,name=cache_lifetime,json=cacheLifetime,proto3" json:"cache_lifetime"`
@ -117,6 +165,20 @@ func (x *Specs) GetSelectorCreated() string {
return ""
}
func (x *Specs) GetCreatedExtractFrom() ExtractFrom {
if x != nil {
return x.CreatedExtractFrom
}
return ExtractFrom_InnerText
}
func (x *Specs) GetCreatedAttributeName() string {
if x != nil {
return x.CreatedAttributeName
}
return ""
}
func (x *Specs) GetSelectorContent() string {
if x != nil {
return x.SelectorContent
@ -144,7 +206,7 @@ var file_proto_specs_proto_rawDesc = string([]byte{
0x0a, 0x11, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x73, 0x70, 0x65, 0x63, 0x73, 0x2e, 0x70, 0x72,
0x6f, 0x74, 0x6f, 0x12, 0x0a, 0x72, 0x73, 0x73, 0x61, 0x6c, 0x63, 0x68, 0x65, 0x6d, 0x79, 0x1a,
0x13, 0x74, 0x61, 0x67, 0x67, 0x65, 0x72, 0x2f, 0x74, 0x61, 0x67, 0x67, 0x65, 0x72, 0x2e, 0x70,
0x72, 0x6f, 0x74, 0x6f, 0x22, 0xef, 0x06, 0x0a, 0x05, 0x53, 0x70, 0x65, 0x63, 0x73, 0x12, 0x30,
0x72, 0x6f, 0x74, 0x6f, 0x22, 0xc0, 0x08, 0x0a, 0x05, 0x53, 0x70, 0x65, 0x63, 0x73, 0x12, 0x30,
0x0a, 0x03, 0x75, 0x72, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x42, 0x1e, 0x9a, 0x84, 0x9e,
0x03, 0x19, 0x6a, 0x73, 0x6f, 0x6e, 0x3a, 0x22, 0x75, 0x72, 0x6c, 0x22, 0x20, 0x76, 0x61, 0x6c,
0x69, 0x64, 0x61, 0x74, 0x65, 0x3a, 0x22, 0x75, 0x72, 0x6c, 0x22, 0x52, 0x03, 0x75, 0x72, 0x6c,
@ -171,37 +233,53 @@ var file_proto_specs_proto_rawDesc = string([]byte{
0x76, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x3a, 0x22, 0x6f, 0x6d, 0x69, 0x74, 0x65, 0x6d,
0x70, 0x74, 0x79, 0x2c, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x22, 0x52, 0x13, 0x73,
0x65, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69,
0x6f, 0x6e, 0x12, 0x58, 0x0a, 0x0f, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x5f, 0x61,
0x75, 0x74, 0x68, 0x6f, 0x72, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x42, 0x2f, 0x9a, 0x84, 0x9e,
0x03, 0x2a, 0x6a, 0x73, 0x6f, 0x6e, 0x3a, 0x22, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72,
0x6f, 0x6e, 0x12, 0x62, 0x0a, 0x0f, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x5f, 0x61,
0x75, 0x74, 0x68, 0x6f, 0x72, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x42, 0x39, 0x9a, 0x84, 0x9e,
0x03, 0x34, 0x6a, 0x73, 0x6f, 0x6e, 0x3a, 0x22, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72,
0x5f, 0x61, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x22, 0x20, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74,
0x65, 0x3a, 0x22, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x22, 0x52, 0x0e, 0x73, 0x65,
0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x41, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x12, 0x5b, 0x0a, 0x10,
0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x5f, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64,
0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x42, 0x30, 0x9a, 0x84, 0x9e, 0x03, 0x2b, 0x6a, 0x73, 0x6f,
0x6e, 0x3a, 0x22, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x5f, 0x63, 0x72, 0x65, 0x61,
0x74, 0x65, 0x64, 0x22, 0x20, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x3a, 0x22, 0x73,
0x65, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x22, 0x52, 0x0f, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74,
0x6f, 0x72, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x12, 0x65, 0x0a, 0x10, 0x73, 0x65, 0x6c,
0x65, 0x63, 0x74, 0x6f, 0x72, 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x18, 0x08, 0x20,
0x01, 0x28, 0x09, 0x42, 0x3a, 0x9a, 0x84, 0x9e, 0x03, 0x35, 0x6a, 0x73, 0x6f, 0x6e, 0x3a, 0x22,
0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74,
0x22, 0x20, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x3a, 0x22, 0x6f, 0x6d, 0x69, 0x74,
0x65, 0x6d, 0x70, 0x74, 0x79, 0x2c, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x22, 0x52,
0x0f, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74,
0x12, 0x61, 0x0a, 0x12, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x5f, 0x65, 0x6e, 0x63,
0x6c, 0x6f, 0x73, 0x75, 0x72, 0x65, 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x42, 0x32, 0x9a, 0x84,
0x9e, 0x03, 0x2d, 0x6a, 0x73, 0x6f, 0x6e, 0x3a, 0x22, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x6f,
0x72, 0x5f, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x73, 0x75, 0x72, 0x65, 0x22, 0x20, 0x76, 0x61, 0x6c,
0x69, 0x64, 0x61, 0x74, 0x65, 0x3a, 0x22, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x22,
0x52, 0x11, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x45, 0x6e, 0x63, 0x6c, 0x6f, 0x73,
0x75, 0x72, 0x65, 0x12, 0x41, 0x0a, 0x0e, 0x63, 0x61, 0x63, 0x68, 0x65, 0x5f, 0x6c, 0x69, 0x66,
0x65, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x42, 0x1a, 0x9a, 0x84, 0x9e,
0x03, 0x15, 0x6a, 0x73, 0x6f, 0x6e, 0x3a, 0x22, 0x63, 0x61, 0x63, 0x68, 0x65, 0x5f, 0x6c, 0x69,
0x66, 0x65, 0x74, 0x69, 0x6d, 0x65, 0x22, 0x52, 0x0d, 0x63, 0x61, 0x63, 0x68, 0x65, 0x4c, 0x69,
0x66, 0x65, 0x74, 0x69, 0x6d, 0x65, 0x42, 0x16, 0x5a, 0x14, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e,
0x61, 0x6c, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x68, 0x74, 0x74, 0x70, 0x2f, 0x70, 0x62, 0x62, 0x06,
0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
0x65, 0x3a, 0x22, 0x6f, 0x6d, 0x69, 0x74, 0x65, 0x6d, 0x70, 0x74, 0x79, 0x2c, 0x73, 0x65, 0x6c,
0x65, 0x63, 0x74, 0x6f, 0x72, 0x22, 0x52, 0x0e, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72,
0x41, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x12, 0x5b, 0x0a, 0x10, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74,
0x6f, 0x72, 0x5f, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09,
0x42, 0x30, 0x9a, 0x84, 0x9e, 0x03, 0x2b, 0x6a, 0x73, 0x6f, 0x6e, 0x3a, 0x22, 0x73, 0x65, 0x6c,
0x65, 0x63, 0x74, 0x6f, 0x72, 0x5f, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x22, 0x20, 0x76,
0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x3a, 0x22, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x6f,
0x72, 0x22, 0x52, 0x0f, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x43, 0x72, 0x65, 0x61,
0x74, 0x65, 0x64, 0x12, 0x6b, 0x0a, 0x14, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x5f, 0x65,
0x78, 0x74, 0x72, 0x61, 0x63, 0x74, 0x5f, 0x66, 0x72, 0x6f, 0x6d, 0x18, 0x0b, 0x20, 0x01, 0x28,
0x0e, 0x32, 0x17, 0x2e, 0x72, 0x73, 0x73, 0x61, 0x6c, 0x63, 0x68, 0x65, 0x6d, 0x79, 0x2e, 0x45,
0x78, 0x74, 0x72, 0x61, 0x63, 0x74, 0x46, 0x72, 0x6f, 0x6d, 0x42, 0x20, 0x9a, 0x84, 0x9e, 0x03,
0x1b, 0x6a, 0x73, 0x6f, 0x6e, 0x3a, 0x22, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x5f, 0x65,
0x78, 0x74, 0x72, 0x61, 0x63, 0x74, 0x5f, 0x66, 0x72, 0x6f, 0x6d, 0x22, 0x52, 0x12, 0x63, 0x72,
0x65, 0x61, 0x74, 0x65, 0x64, 0x45, 0x78, 0x74, 0x72, 0x61, 0x63, 0x74, 0x46, 0x72, 0x6f, 0x6d,
0x12, 0x58, 0x0a, 0x16, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x5f, 0x61, 0x74, 0x74, 0x72,
0x69, 0x62, 0x75, 0x74, 0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x09,
0x42, 0x22, 0x9a, 0x84, 0x9e, 0x03, 0x1d, 0x6a, 0x73, 0x6f, 0x6e, 0x3a, 0x22, 0x63, 0x72, 0x65,
0x61, 0x74, 0x65, 0x64, 0x5f, 0x61, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, 0x5f, 0x6e,
0x61, 0x6d, 0x65, 0x22, 0x52, 0x14, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74, 0x74,
0x72, 0x69, 0x62, 0x75, 0x74, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x65, 0x0a, 0x10, 0x73, 0x65,
0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x18, 0x08,
0x20, 0x01, 0x28, 0x09, 0x42, 0x3a, 0x9a, 0x84, 0x9e, 0x03, 0x35, 0x6a, 0x73, 0x6f, 0x6e, 0x3a,
0x22, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e,
0x74, 0x22, 0x20, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x3a, 0x22, 0x6f, 0x6d, 0x69,
0x74, 0x65, 0x6d, 0x70, 0x74, 0x79, 0x2c, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x22,
0x52, 0x0f, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x6e,
0x74, 0x12, 0x61, 0x0a, 0x12, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x5f, 0x65, 0x6e,
0x63, 0x6c, 0x6f, 0x73, 0x75, 0x72, 0x65, 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x42, 0x32, 0x9a,
0x84, 0x9e, 0x03, 0x2d, 0x6a, 0x73, 0x6f, 0x6e, 0x3a, 0x22, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74,
0x6f, 0x72, 0x5f, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x73, 0x75, 0x72, 0x65, 0x22, 0x20, 0x76, 0x61,
0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x3a, 0x22, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72,
0x22, 0x52, 0x11, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x45, 0x6e, 0x63, 0x6c, 0x6f,
0x73, 0x75, 0x72, 0x65, 0x12, 0x41, 0x0a, 0x0e, 0x63, 0x61, 0x63, 0x68, 0x65, 0x5f, 0x6c, 0x69,
0x66, 0x65, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x42, 0x1a, 0x9a, 0x84,
0x9e, 0x03, 0x15, 0x6a, 0x73, 0x6f, 0x6e, 0x3a, 0x22, 0x63, 0x61, 0x63, 0x68, 0x65, 0x5f, 0x6c,
0x69, 0x66, 0x65, 0x74, 0x69, 0x6d, 0x65, 0x22, 0x52, 0x0d, 0x63, 0x61, 0x63, 0x68, 0x65, 0x4c,
0x69, 0x66, 0x65, 0x74, 0x69, 0x6d, 0x65, 0x2a, 0x2b, 0x0a, 0x0b, 0x45, 0x78, 0x74, 0x72, 0x61,
0x63, 0x74, 0x46, 0x72, 0x6f, 0x6d, 0x12, 0x0d, 0x0a, 0x09, 0x49, 0x6e, 0x6e, 0x65, 0x72, 0x54,
0x65, 0x78, 0x74, 0x10, 0x00, 0x12, 0x0d, 0x0a, 0x09, 0x41, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75,
0x74, 0x65, 0x10, 0x01, 0x42, 0x16, 0x5a, 0x14, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c,
0x2f, 0x61, 0x70, 0x69, 0x2f, 0x68, 0x74, 0x74, 0x70, 0x2f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72,
0x6f, 0x74, 0x6f, 0x33,
})
var (
@ -216,16 +294,19 @@ func file_proto_specs_proto_rawDescGZIP() []byte {
return file_proto_specs_proto_rawDescData
}
var file_proto_specs_proto_enumTypes = make([]protoimpl.EnumInfo, 1)
var file_proto_specs_proto_msgTypes = make([]protoimpl.MessageInfo, 1)
var file_proto_specs_proto_goTypes = []any{
(*Specs)(nil), // 0: rssalchemy.Specs
(ExtractFrom)(0), // 0: rssalchemy.ExtractFrom
(*Specs)(nil), // 1: rssalchemy.Specs
}
var file_proto_specs_proto_depIdxs = []int32{
0, // [0:0] is the sub-list for method output_type
0, // [0:0] is the sub-list for method input_type
0, // [0:0] is the sub-list for extension type_name
0, // [0:0] is the sub-list for extension extendee
0, // [0:0] is the sub-list for field type_name
0, // 0: rssalchemy.Specs.created_extract_from:type_name -> rssalchemy.ExtractFrom
1, // [1:1] is the sub-list for method output_type
1, // [1:1] is the sub-list for method input_type
1, // [1:1] is the sub-list for extension type_name
1, // [1:1] is the sub-list for extension extendee
0, // [0:1] is the sub-list for field type_name
}
func init() { file_proto_specs_proto_init() }
@ -238,13 +319,14 @@ func file_proto_specs_proto_init() {
File: protoimpl.DescBuilder{
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
RawDescriptor: unsafe.Slice(unsafe.StringData(file_proto_specs_proto_rawDesc), len(file_proto_specs_proto_rawDesc)),
NumEnums: 0,
NumEnums: 1,
NumMessages: 1,
NumExtensions: 0,
NumServices: 0,
},
GoTypes: file_proto_specs_proto_goTypes,
DependencyIndexes: file_proto_specs_proto_depIdxs,
EnumInfos: file_proto_specs_proto_enumTypes,
MessageInfos: file_proto_specs_proto_msgTypes,
}.Build()
File_proto_specs_proto = out.File

View File

@ -103,10 +103,11 @@ func (p *pageParser) extractPost(post playwright.Locator) (models.FeedItem, erro
item.Description = newLocator(post, p.task.SelectorDescription).First().InnerText()
}
if len(p.task.SelectorAuthor) > 0 {
item.AuthorName = newLocator(post, p.task.SelectorAuthor).First().InnerText()
item.AuthorLink = newLocator(post, p.task.SelectorAuthor).First().GetAttribute("href")
item.AuthorLink = absUrl(item.AuthorLink, page)
}
if len(p.task.SelectorContent) > 0 {
item.Content = p.extractContent(post)
@ -114,7 +115,15 @@ func (p *pageParser) extractPost(post playwright.Locator) (models.FeedItem, erro
item.Enclosure = newLocator(post, p.task.SelectorEnclosure).First().GetAttribute("src")
createdDateStr := newLocator(post, p.task.SelectorCreated).First().InnerText()
var createdDateStr string
switch p.task.CreatedExtractFrom {
case models.ExtractFrom_InnerText:
createdDateStr = newLocator(post, p.task.SelectorCreated).First().InnerText()
case models.ExtractFrom_Attribute:
createdDateStr = newLocator(post, p.task.SelectorCreated).First().GetAttribute(p.task.CreatedAttributeName)
default:
return models.FeedItem{}, fmt.Errorf("invalid task.CreatedExtractFrom")
}
log.Debugf("date=%s", createdDateStr)
createdDate, err := p.dateParser.ParseDate(createdDateStr)
if err != nil {

View File

@ -13,6 +13,13 @@ const (
TaskTypePageScreenshot = "page_screenshot"
)
type ExtractFrom int
const (
ExtractFrom_InnerText ExtractFrom = 0
ExtractFrom_Attribute ExtractFrom = 1
)
type Task struct {
// While adding new fields, dont forget to alter caching func
TaskType TaskType
@ -23,6 +30,8 @@ type Task struct {
SelectorDescription string
SelectorAuthor string
SelectorCreated string
CreatedExtractFrom ExtractFrom
CreatedAttributeName string
SelectorContent string
SelectorEnclosure string
Headers map[string]string

View File

@ -6,14 +6,23 @@ import "tagger/tagger.proto";
option go_package = "internal/api/http/pb";
enum ExtractFrom {
InnerText = 0;
Attribute = 1;
}
message Specs {
string url = 1 [(tagger.tags) = "json:\"url\" validate:\"url\""];
string selector_post = 2 [(tagger.tags) = "json:\"selector_post\" validate:\"selector\""];
string selector_title = 3 [(tagger.tags) = "json:\"selector_title\" validate:\"selector\""];
string selector_link = 4 [(tagger.tags) = "json:\"selector_link\" validate:\"selector\""];
string selector_description = 5 [(tagger.tags) = "json:\"selector_description\" validate:\"omitempty,selector\""];
string selector_author = 6 [(tagger.tags) = "json:\"selector_author\" validate:\"selector\""];
string selector_author = 6 [(tagger.tags) = "json:\"selector_author\" validate:\"omitempty,selector\""];
string selector_created = 7 [(tagger.tags) = "json:\"selector_created\" validate:\"selector\""];
ExtractFrom created_extract_from = 11 [(tagger.tags) = "json:\"created_extract_from\""];
string created_attribute_name = 12 [(tagger.tags) = "json:\"created_attribute_name\""];
string selector_content = 8 [(tagger.tags) = "json:\"selector_content\" validate:\"omitempty,selector\""];
string selector_enclosure = 9 [(tagger.tags) = "json:\"selector_enclosure\" validate:\"selector\""];
string cache_lifetime = 10 [(tagger.tags) = "json:\"cache_lifetime\""];