egor3f/rssalchemy
1
0
Fork 0
Code Actions Packages Releases Wiki Activity

Compare commits

base: egor3f:0e32cc3f176baa3314e08b9eaf6a565e0b61f0fe
Branches Tags
egor3f:master
egor3f:extract-attrs2
egor3f:multithread
egor3f:adblock2
egor3f:adblock1
egor3f:vue
..
compare: egor3f:3e53958a35cfb8f98de92d12971aa645d30bbbd1
Branches Tags
egor3f:master
egor3f:extract-attrs2
egor3f:multithread
egor3f:adblock2
egor3f:adblock1
egor3f:vue

No commits in common. "0e32cc3f176baa3314e08b9eaf6a565e0b61f0fe" and "3e53958a35cfb8f98de92d12971aa645d30bbbd1" have entirely different histories.
0e32cc3f17 ... 3e53958a35

15 changed files with 155 additions and 421 deletions
Show Stats Expand all files Collapse all files

5
frontend/wizard-vue/src/common/enum.ts
View File

@ -1,5 +0,0 @@
export type EnumValue = {
label: string
value: number
}
export type Enum = EnumValue[]

25
frontend/wizard-vue/src/components/EditUrlModal.vue
View File

@ -1,10 +1,20 @@
<script setup lang="ts"> <script setup lang="ts">
import TextField from "@/components/inputs/TextField.vue"; import Field from "@/components/Field.vue";
import {type Field as FieldSpec} from "@/urlmaker/specs";
import {validateOr, validatePreset, validateUrl} from "@/urlmaker/validators.ts";
import Btn from "@/components/Btn.vue"; import Btn from "@/components/Btn.vue";
import {onMounted, onUnmounted, ref, watch} from "vue"; import {onMounted, onUnmounted, ref, watch} from "vue";
import Modal from "@/components/Modal.vue"; import Modal from "@/components/Modal.vue";
import {validatePreset, validateUrl} from "@/urlmaker/validators.ts";
const field: FieldSpec = {
name: '',
input_type: 'url',
label: 'URL of feed or preset',
default: '',
required: true,
validate: validateOr(validateUrl, validatePreset),
}
const visible = defineModel('visible', { const visible = defineModel('visible', {
type: Boolean, type: Boolean,
@ -23,10 +33,11 @@ watch(visible, () => {
const valid = ref(false); const valid = ref(false);
watch(url, (value) => { watch(url, (value) => {
valid.value = validateUrl(value) || validatePreset(value); valid.value = field.validate(value).ok;
}); });
const accept = () => { const accept = () => {
valid.value = field.validate(url.value).ok;
if (valid.value) { if (valid.value) {
emit('update:modelValue', url.value); emit('update:modelValue', url.value);
emit('update:visible', false); emit('update:visible', false);
@ -48,13 +59,7 @@ onUnmounted(() => {
<template> <template>
<Modal v-model="visible"> <Modal v-model="visible">
<TextField <Field :field="field" v-model="url" :focused="true"/>
name="url"
input_type="url"
label="URL of feed or preset"
v-model="url"
:focused="true"
/>
<Btn :active="valid" @click="accept">Edit</Btn> <Btn :active="valid" @click="accept">Edit</Btn>
</Modal> </Modal>
</template> </template>

11
frontend/wizard-vue/src/components/inputs/TextField.vue → frontend/wizard-vue/src/components/Field.vue
View File

@ -1,10 +1,9 @@
<script setup lang="ts"> <script setup lang="ts">
import type {Field} from "@/urlmaker/specs.ts";
import {getCurrentInstance, onMounted, useTemplateRef} from "vue"; import {getCurrentInstance, onMounted, useTemplateRef} from "vue";
const {name, label, input_type, focused} = defineProps<{ const {field, focused} = defineProps<{
name: string field: Field,
label: string,
input_type: 'text' | 'url',
focused?: boolean, focused?: boolean,
}>(); }>();
const id = 'field' + getCurrentInstance()?.uid; const id = 'field' + getCurrentInstance()?.uid;
@ -19,9 +18,9 @@ onMounted(() => {
<template> <template>
<div class="field"> <div class="field">
<div class="label"><label :for="id">{{ label }}</label></div> <div class="label"><label :for="id">{{ field.label }}</label></div>
<div class="input"> <div class="input">
<input :type="input_type" :name="name" :id="id" v-model="model" ref="field"/> <input :type="field.input_type" :name="field.name" :id="id" v-model="model" ref="field"/>
</div> </div>
</div> </div>
</template> </template>

45
frontend/wizard-vue/src/components/SpecsForm.vue
View File

@ -1,51 +1,22 @@
<script setup lang="ts"> <script setup lang="ts">
import {fields, InputType, type SpecField} from '@/urlmaker/specs.ts'; import {fields, type Specs} from '@/urlmaker/specs.ts';
import TextField from "@/components/inputs/TextField.vue"; import Field from "@/components/Field.vue";
import RadioButtons from "@/components/inputs/RadioButtons.vue";
import {useWizardStore} from "@/stores/wizard.ts"; import {useWizardStore} from "@/stores/wizard.ts";
const store = useWizardStore(); const store = useWizardStore();
const groups: SpecField[][] = [];
for (const field of fields) {
if(groups.length === 0 || groups[groups.length - 1][0].group != field.group) {
groups.push([field]);
} else {
groups[groups.length - 1].push(field);
}
}
</script> </script>
<template> <template>
<div> <div>
<div class="group" v-for="group in Object.values(groups)"> <Field v-for="field in fields"
<template v-for="field in group"> :field="field"
<TextField :model-value="store.specs[field.name]"
v-if="field.input_type === InputType.Url || field.input_type === InputType.Text" @update:model-value="event => store.updateSpec(field.name, event)"
v-show="!field.show_if || field.show_if(store.specs)" ></Field>
:name="field.name"
:label="field.label"
:input_type="field.input_type"
:model-value="store.specs[field.name]"
@update:model-value="event => store.updateSpec(field.name, event)"
></TextField>
<RadioButtons
v-if="field.input_type === InputType.Radio"
v-show="!field.show_if || field.show_if(store.specs)"
:name="field.name"
:label="field.label"
:values="field.enum!"
:model-value="store.specs[field.name]"
@update:model-value="event => store.updateSpec(field.name, event)"
></RadioButtons>
</template>
</div>
</div> </div>
</template> </template>
<style scoped lang="scss"> <style scoped lang="scss">
div.group {
margin: 20px 0;
}
</style> </style>

51
frontend/wizard-vue/src/components/inputs/RadioButtons.vue
View File

@ -1,51 +0,0 @@
<script setup lang="ts">
import {getCurrentInstance} from "vue";
import type {Enum} from "@/common/enum.ts";
const {name, label, values} = defineProps<{
name: string
label: string,
values: Enum,
}>();
const componentId = 'field' + getCurrentInstance()?.uid;
const model = defineModel();
</script>
<template>
<div class="field">
<span class="field-label"><label>{{ label }}</label></span>
<template class="value" v-for="enumValue in values">
<input
type="radio"
:name="name"
:value="enumValue.value"
:id="`${componentId}_${enumValue.value}`"
v-model="model"
/>
<label class="radio-label" :for="`${componentId}_${enumValue.value}`">{{ enumValue.label }}</label>
</template>
</div>
</template>
<style scoped lang="scss">
div.field {
margin: 0 0 8px 0;
}
.field-label {
font-size: 0.9em;
margin-right: 8px;
}
.radio-label {
font-size: 0.9em;
}
input, .radio-label, .field-label {
vertical-align: middle;
}
input {
margin-top: 0;
}
</style>

5
frontend/wizard-vue/src/pages/WizardPage.vue
View File

@ -19,8 +19,8 @@ watch(existingLink, async (value) => {
if(!value) return; if(!value) return;
existingLink.value = ""; existingLink.value = "";
try { try {
if(validateUrl(value)) store.updateSpecs(await decodeUrl(value)); if(validateUrl(value).ok) store.updateSpecs(await decodeUrl(value));
else if (validatePreset(value)) store.updateSpecs(await decodePreset(value)); else if (validatePreset(value).ok) store.updateSpecs(await decodePreset(value));
} catch (e) { } catch (e) {
console.log(e); console.log(e);
alert(`Decoding error: ${e}`); alert(`Decoding error: ${e}`);
@ -60,6 +60,7 @@ function screenshot() {
<template> <template>
<div class="wrapper"> <div class="wrapper">
<SpecsForm class="specs-form"></SpecsForm> <SpecsForm class="specs-form"></SpecsForm>
<!-- <Btn :active="store.formValid" @click="generateLink">Generate link</Btn>-->
<Btn :active="store.formValid" @click="screenshot">Screenshot</Btn> <Btn :active="store.formValid" @click="screenshot">Screenshot</Btn>
<Btn @click="editModalVisible = true">Edit existing task / import preset</Btn> <Btn @click="editModalVisible = true">Edit existing task / import preset</Btn>
<Btn @click="store.reset">Reset Form</Btn> <Btn @click="store.reset">Reset Form</Btn>

6
frontend/wizard-vue/src/stores/wizard.ts
View File

@ -1,5 +1,5 @@
import {defineStore} from "pinia"; import {defineStore} from "pinia";
import {emptySpecs, type SpecField, fields, type Specs, type SpecValue} from "@/urlmaker/specs.ts"; import {emptySpecs, type Field, type FieldNames, fields, type Specs} from "@/urlmaker/specs.ts";
import {computed, reactive} from "vue"; import {computed, reactive} from "vue";
import {debounce} from "es-toolkit"; import {debounce} from "es-toolkit";
@ -14,7 +14,7 @@ export const useWizardStore = defineStore('wizard', () => {
const formValid = computed(() => { const formValid = computed(() => {
return fields.every(field => ( return fields.every(field => (
!specs[field.name] && !(field as SpecField).required || field.validate(specs[field.name]!) specs[field.name].length === 0 && !(field as Field).required || field.validate(specs[field.name]).ok
)); ));
}); });
@ -22,7 +22,7 @@ export const useWizardStore = defineStore('wizard', () => {
localStorage.setItem(LOCAL_STORAGE_KEY, JSON.stringify(specs)); localStorage.setItem(LOCAL_STORAGE_KEY, JSON.stringify(specs));
}, 100); }, 100);
function updateSpec(fieldName: keyof Specs, newValue: SpecValue) { function updateSpec(fieldName: FieldNames, newValue: string) {
specs[fieldName] = newValue; specs[fieldName] = newValue;
updateLocalStorage(); updateLocalStorage();
} }

50
frontend/wizard-vue/src/urlmaker/proto/specs.ts
View File

@ -5,10 +5,6 @@
* git: https://github.com/thesayyn/protoc-gen-ts */ * git: https://github.com/thesayyn/protoc-gen-ts */
import * as pb_1 from "google-protobuf"; import * as pb_1 from "google-protobuf";
export namespace rssalchemy { export namespace rssalchemy {
export enum ExtractFrom {
InnerText = 0,
Attribute = 1
}
export class Specs extends pb_1.Message { export class Specs extends pb_1.Message {
#one_of_decls: number[][] = []; #one_of_decls: number[][] = [];
constructor(data?: any[] | { constructor(data?: any[] | {
@ -19,8 +15,6 @@ export namespace rssalchemy {
selector_description?: string; selector_description?: string;
selector_author?: string; selector_author?: string;
selector_created?: string; selector_created?: string;
created_extract_from?: ExtractFrom;
created_attribute_name?: string;
selector_content?: string; selector_content?: string;
selector_enclosure?: string; selector_enclosure?: string;
cache_lifetime?: string; cache_lifetime?: string;
@ -49,12 +43,6 @@ export namespace rssalchemy {
if ("selector_created" in data && data.selector_created != undefined) { if ("selector_created" in data && data.selector_created != undefined) {
this.selector_created = data.selector_created; this.selector_created = data.selector_created;
} }
if ("created_extract_from" in data && data.created_extract_from != undefined) {
this.created_extract_from = data.created_extract_from;
}
if ("created_attribute_name" in data && data.created_attribute_name != undefined) {
this.created_attribute_name = data.created_attribute_name;
}
if ("selector_content" in data && data.selector_content != undefined) { if ("selector_content" in data && data.selector_content != undefined) {
this.selector_content = data.selector_content; this.selector_content = data.selector_content;
} }
@ -108,18 +96,6 @@ export namespace rssalchemy {
set selector_created(value: string) { set selector_created(value: string) {
pb_1.Message.setField(this, 7, value); pb_1.Message.setField(this, 7, value);
} }
get created_extract_from() {
return pb_1.Message.getFieldWithDefault(this, 11, ExtractFrom.InnerText) as ExtractFrom;
}
set created_extract_from(value: ExtractFrom) {
pb_1.Message.setField(this, 11, value);
}
get created_attribute_name() {
return pb_1.Message.getFieldWithDefault(this, 12, "") as string;
}
set created_attribute_name(value: string) {
pb_1.Message.setField(this, 12, value);
}
get selector_content() { get selector_content() {
return pb_1.Message.getFieldWithDefault(this, 8, "") as string; return pb_1.Message.getFieldWithDefault(this, 8, "") as string;
} }
@ -146,8 +122,6 @@ export namespace rssalchemy {
selector_description?: string; selector_description?: string;
selector_author?: string; selector_author?: string;
selector_created?: string; selector_created?: string;
created_extract_from?: ExtractFrom;
created_attribute_name?: string;
selector_content?: string; selector_content?: string;
selector_enclosure?: string; selector_enclosure?: string;
cache_lifetime?: string; cache_lifetime?: string;
@ -174,12 +148,6 @@ export namespace rssalchemy {
if (data.selector_created != null) { if (data.selector_created != null) {
message.selector_created = data.selector_created; message.selector_created = data.selector_created;
} }
if (data.created_extract_from != null) {
message.created_extract_from = data.created_extract_from;
}
if (data.created_attribute_name != null) {
message.created_attribute_name = data.created_attribute_name;
}
if (data.selector_content != null) { if (data.selector_content != null) {
message.selector_content = data.selector_content; message.selector_content = data.selector_content;
} }
@ -200,8 +168,6 @@ export namespace rssalchemy {
selector_description?: string; selector_description?: string;
selector_author?: string; selector_author?: string;
selector_created?: string; selector_created?: string;
created_extract_from?: ExtractFrom;
created_attribute_name?: string;
selector_content?: string; selector_content?: string;
selector_enclosure?: string; selector_enclosure?: string;
cache_lifetime?: string; cache_lifetime?: string;
@ -227,12 +193,6 @@ export namespace rssalchemy {
if (this.selector_created != null) { if (this.selector_created != null) {
data.selector_created = this.selector_created; data.selector_created = this.selector_created;
} }
if (this.created_extract_from != null) {
data.created_extract_from = this.created_extract_from;
}
if (this.created_attribute_name != null) {
data.created_attribute_name = this.created_attribute_name;
}
if (this.selector_content != null) { if (this.selector_content != null) {
data.selector_content = this.selector_content; data.selector_content = this.selector_content;
} }
@ -262,10 +222,6 @@ export namespace rssalchemy {
writer.writeString(6, this.selector_author); writer.writeString(6, this.selector_author);
if (this.selector_created.length) if (this.selector_created.length)
writer.writeString(7, this.selector_created); writer.writeString(7, this.selector_created);
if (this.created_extract_from != ExtractFrom.InnerText)
writer.writeEnum(11, this.created_extract_from);
if (this.created_attribute_name.length)
writer.writeString(12, this.created_attribute_name);
if (this.selector_content.length) if (this.selector_content.length)
writer.writeString(8, this.selector_content); writer.writeString(8, this.selector_content);
if (this.selector_enclosure.length) if (this.selector_enclosure.length)
@ -302,12 +258,6 @@ export namespace rssalchemy {
case 7: case 7:
message.selector_created = reader.readString(); message.selector_created = reader.readString();
break; break;
case 11:
message.created_extract_from = reader.readEnum();
break;
case 12:
message.created_attribute_name = reader.readString();
break;
case 8: case 8:
message.selector_content = reader.readString(); message.selector_content = reader.readString();
break; break;

78
frontend/wizard-vue/src/urlmaker/specs.ts
View File

@ -1,39 +1,23 @@
import { import {
validateAttribute,
validateDuration, validateDuration,
validateSelector, validateSelector,
validateUrl, validateUrl,
type validator type validator
} from "@/urlmaker/validators.ts"; } from "@/urlmaker/validators.ts";
import {rssalchemy} from "@/urlmaker/proto/specs.ts";
import type {Enum} from "@/common/enum.ts";
export type SpecKey = ReturnType<rssalchemy.Specs['toObject']>; export interface Field {
export type SpecValue = string | number; name: string
export type Specs = {[k in keyof SpecKey]: SpecValue}; input_type: string
export enum InputType {
Url = 'url',
Text = 'text',
Radio = 'radio'
}
export interface SpecField {
name: keyof Specs
input_type: InputType
enum?: Enum,
label: string label: string
default: SpecValue default: string
validate: validator validate: validator
required?: boolean required?: boolean
group?: string
show_if?: (specs: Specs) => boolean
} }
export const fields: SpecField[] = [ export const fields = [
{ {
name: 'url', name: 'url',
input_type: InputType.Url, input_type: 'url',
label: 'URL of page for converting', label: 'URL of page for converting',
default: '', default: '',
validate: validateUrl, validate: validateUrl,
@ -41,94 +25,72 @@ export const fields: SpecField[] = [
}, },
{ {
name: 'selector_post', name: 'selector_post',
input_type: InputType.Text, input_type: 'text',
label: 'CSS Selector for post', label: 'CSS Selector for post',
default: '', default: '',
validate: validateSelector, validate: validateSelector,
}, },
{ {
name: 'selector_title', name: 'selector_title',
input_type: InputType.Text, input_type: 'text',
label: 'CSS Selector for title', label: 'CSS Selector for title',
default: '', default: '',
validate: validateSelector, validate: validateSelector,
}, },
{ {
name: 'selector_link', name: 'selector_link',
input_type: InputType.Text, input_type: 'text',
label: 'CSS Selector for link', label: 'CSS Selector for link',
default: '', default: '',
validate: validateSelector, validate: validateSelector,
}, },
{ {
name: 'selector_description', name: 'selector_description',
input_type: InputType.Text, input_type: 'text',
label: 'CSS Selector for description', label: 'CSS Selector for description',
default: '', default: '',
validate: validateSelector, validate: validateSelector,
}, },
{ {
name: 'selector_author', name: 'selector_author',
input_type: InputType.Text, input_type: 'text',
label: 'CSS Selector for author', label: 'CSS Selector for author',
default: '', default: '',
validate: validateSelector, validate: validateSelector,
}, },
{ {
name: 'selector_created', name: 'selector_created',
input_type: InputType.Text, input_type: 'text',
label: 'CSS Selector for created date', label: 'CSS Selector for created date',
default: '', default: '',
validate: validateSelector, validate: validateSelector,
group: 'created',
}, },
{
name: 'created_extract_from',
input_type: InputType.Radio,
enum: [
{label: 'Inner Text', value: rssalchemy.ExtractFrom.InnerText},
{label: 'Attribute', value: rssalchemy.ExtractFrom.Attribute},
],
label: 'Extract from',
default: rssalchemy.ExtractFrom.InnerText,
validate: value => Object.values(rssalchemy.ExtractFrom).includes(value),
group: 'created',
show_if: specs => !!specs.selector_created,
},
{
name: 'created_attribute_name',
input_type: InputType.Text,
label: 'Attribute name',
default: '',
validate: validateAttribute,
show_if: specs =>
!!specs.selector_created && specs.created_extract_from === rssalchemy.ExtractFrom.Attribute,
group: 'created',
},
{ {
name: 'selector_content', name: 'selector_content',
input_type: InputType.Text, input_type: 'text',
label: 'CSS Selector for content', label: 'CSS Selector for content',
default: '', default: '',
validate: validateSelector, validate: validateSelector,
}, },
{ {
name: 'selector_enclosure', name: 'selector_enclosure',
input_type: InputType.Text, input_type: 'text',
label: 'CSS Selector for enclosure (e.g. image url)', label: 'CSS Selector for enclosure (e.g. image url)',
default: '', default: '',
validate: validateSelector, validate: validateSelector,
}, },
{ {
name: 'cache_lifetime', name: 'cache_lifetime',
input_type: InputType.Text, input_type: 'text',
label: 'Cache lifetime (format examples: 10s, 1m, 2h)', label: 'Cache lifetime (format examples: 10s, 1m, 2h)',
default: '10m', default: '10m',
validate: validateDuration, validate: validateDuration,
}, },
]; ] as const satisfies Field[];
export type FieldNames = (typeof fields)[number]['name'];
export type Specs = {[k in FieldNames]: string};
export const emptySpecs = fields.reduce((o, f) => { export const emptySpecs = fields.reduce((o, f) => {
o[f.name] = f.default; o[f.name] = f.default;

55
frontend/wizard-vue/src/urlmaker/validators.ts
View File

@ -1,35 +1,54 @@
import {presetPrefix} from "@/urlmaker/index.ts"; import {presetPrefix} from "@/urlmaker/index.ts";
import type {SpecValue} from "@/urlmaker/specs.ts";
export type validator = (v: SpecValue) => boolean; type validResult = { ok: boolean, error?: string };
export type validator = (v: string) => validResult
export function validateUrl(s: SpecValue): boolean { export function validateUrl(s: string): validResult {
let url; let url;
try { try {
url = new URL(s as string); url = new URL(s);
return url.protocol === "http:" || url.protocol === "https:" return {
ok: url.protocol === "http:" || url.protocol === "https:",
error: 'Invalid URL protocol',
};
} catch { } catch {
return false; return {ok: false, error: 'Invalid URL'};
} }
} }
export function validatePreset(s: SpecValue): boolean { export function validatePreset(s: string): validResult {
return (s as string).startsWith(presetPrefix); if(!s.startsWith(presetPrefix)) {
return {
ok: false,
error: 'Not a preset'
}
}
return {ok: true}
} }
export function validateSelector(s: SpecValue): boolean { export function validateOr(...validators: validator[]): validator {
return function(s: string): validResult {
return validators.reduce<validResult>((res, v) => {
let r = v(s);
if(r.ok) res.ok = true;
else res.error += r.error + '; ';
return res;
}, {ok: false, error: ''});
}
}
export function validateSelector(s: string): validResult {
try { try {
document.createDocumentFragment().querySelector(s as string); document.createDocumentFragment().querySelector(s);
return true; return {ok: true}
} catch { } catch {
return false; return {ok: false, error: 'Invalid selector'};
} }
} }
export function validateAttribute(s: SpecValue): boolean { export function validateDuration(s: string): validResult {
return /([^\t\n\f \/>"'=]+)/.test(s as string); return {
} ok: /^\d+[smh]$/.test(s),
error: 'Duration must be number and unit (s/m/h), example: 5s = 5 seconds'
export function validateDuration(s: SpecValue): boolean { }
return /^\d+[smh]$/.test(s as string);
} }

32
internal/api/http/handler.go
View File

@ -75,28 +75,18 @@ func (h *Handler) handleRender(c echo.Context) error {
return echo.NewHTTPError(400, fmt.Errorf("decode specs: %w", err)) return echo.NewHTTPError(400, fmt.Errorf("decode specs: %w", err))
} }
extractFrom, ok := map[pb.ExtractFrom]models.ExtractFrom{
pb.ExtractFrom_InnerText: models.ExtractFrom_InnerText,
pb.ExtractFrom_Attribute: models.ExtractFrom_Attribute,
}[specs.CreatedExtractFrom]
if !ok {
return echo.NewHTTPError(400, "invalid extract from")
}
task := models.Task{ task := models.Task{
TaskType: models.TaskTypeExtract, TaskType: models.TaskTypeExtract,
URL: specs.Url, URL: specs.Url,
SelectorPost: specs.SelectorPost, SelectorPost: specs.SelectorPost,
SelectorTitle: specs.SelectorTitle, SelectorTitle: specs.SelectorTitle,
SelectorLink: specs.SelectorLink, SelectorLink: specs.SelectorLink,
SelectorDescription: specs.SelectorDescription, SelectorDescription: specs.SelectorDescription,
SelectorAuthor: specs.SelectorAuthor, SelectorAuthor: specs.SelectorAuthor,
SelectorCreated: specs.SelectorCreated, SelectorCreated: specs.SelectorCreated,
CreatedExtractFrom: extractFrom, SelectorContent: specs.SelectorContent,
CreatedAttributeName: specs.CreatedAttributeName, SelectorEnclosure: specs.SelectorEnclosure,
SelectorContent: specs.SelectorContent, Headers: extractHeaders(c),
SelectorEnclosure: specs.SelectorEnclosure,
Headers: extractHeaders(c),
} }
cacheLifetime, err := time.ParseDuration(specs.CacheLifetime) cacheLifetime, err := time.ParseDuration(specs.CacheLifetime)

163
internal/api/http/pb/specs.pb.go
View File

@ -22,68 +22,20 @@ const (
_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
) )
type ExtractFrom int32
const (
ExtractFrom_InnerText ExtractFrom = 0
ExtractFrom_Attribute ExtractFrom = 1
)
// Enum value maps for ExtractFrom.
var (
ExtractFrom_name = map[int32]string{
0: "InnerText",
1: "Attribute",
}
ExtractFrom_value = map[string]int32{
"InnerText": 0,
"Attribute": 1,
}
)
func (x ExtractFrom) Enum() *ExtractFrom {
p := new(ExtractFrom)
*p = x
return p
}
func (x ExtractFrom) String() string {
return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x))
}
func (ExtractFrom) Descriptor() protoreflect.EnumDescriptor {
return file_proto_specs_proto_enumTypes[0].Descriptor()
}
func (ExtractFrom) Type() protoreflect.EnumType {
return &file_proto_specs_proto_enumTypes[0]
}
func (x ExtractFrom) Number() protoreflect.EnumNumber {
return protoreflect.EnumNumber(x)
}
// Deprecated: Use ExtractFrom.Descriptor instead.
func (ExtractFrom) EnumDescriptor() ([]byte, []int) {
return file_proto_specs_proto_rawDescGZIP(), []int{0}
}
type Specs struct { type Specs struct {
state protoimpl.MessageState `protogen:"open.v1"` state protoimpl.MessageState `protogen:"open.v1"`
Url string `protobuf:"bytes,1,opt,name=url,proto3" json:"url" validate:"url"` Url string `protobuf:"bytes,1,opt,name=url,proto3" json:"url" validate:"url"`
SelectorPost string `protobuf:"bytes,2,opt,name=selector_post,json=selectorPost,proto3" json:"selector_post" validate:"selector"` SelectorPost string `protobuf:"bytes,2,opt,name=selector_post,json=selectorPost,proto3" json:"selector_post" validate:"selector"`
SelectorTitle string `protobuf:"bytes,3,opt,name=selector_title,json=selectorTitle,proto3" json:"selector_title" validate:"selector"` SelectorTitle string `protobuf:"bytes,3,opt,name=selector_title,json=selectorTitle,proto3" json:"selector_title" validate:"selector"`
SelectorLink string `protobuf:"bytes,4,opt,name=selector_link,json=selectorLink,proto3" json:"selector_link" validate:"selector"` SelectorLink string `protobuf:"bytes,4,opt,name=selector_link,json=selectorLink,proto3" json:"selector_link" validate:"selector"`
SelectorDescription string `protobuf:"bytes,5,opt,name=selector_description,json=selectorDescription,proto3" json:"selector_description" validate:"omitempty,selector"` SelectorDescription string `protobuf:"bytes,5,opt,name=selector_description,json=selectorDescription,proto3" json:"selector_description" validate:"omitempty,selector"`
SelectorAuthor string `protobuf:"bytes,6,opt,name=selector_author,json=selectorAuthor,proto3" json:"selector_author" validate:"selector"` SelectorAuthor string `protobuf:"bytes,6,opt,name=selector_author,json=selectorAuthor,proto3" json:"selector_author" validate:"selector"`
SelectorCreated string `protobuf:"bytes,7,opt,name=selector_created,json=selectorCreated,proto3" json:"selector_created" validate:"selector"` SelectorCreated string `protobuf:"bytes,7,opt,name=selector_created,json=selectorCreated,proto3" json:"selector_created" validate:"selector"`
CreatedExtractFrom ExtractFrom `protobuf:"varint,11,opt,name=created_extract_from,json=createdExtractFrom,proto3,enum=rssalchemy.ExtractFrom" json:"created_extract_from"` SelectorContent string `protobuf:"bytes,8,opt,name=selector_content,json=selectorContent,proto3" json:"selector_content" validate:"omitempty,selector"`
CreatedAttributeName string `protobuf:"bytes,12,opt,name=created_attribute_name,json=createdAttributeName,proto3" json:"created_attribute_name"` SelectorEnclosure string `protobuf:"bytes,9,opt,name=selector_enclosure,json=selectorEnclosure,proto3" json:"selector_enclosure" validate:"selector"`
SelectorContent string `protobuf:"bytes,8,opt,name=selector_content,json=selectorContent,proto3" json:"selector_content" validate:"omitempty,selector"` CacheLifetime string `protobuf:"bytes,10,opt,name=cache_lifetime,json=cacheLifetime,proto3" json:"cache_lifetime"`
SelectorEnclosure string `protobuf:"bytes,9,opt,name=selector_enclosure,json=selectorEnclosure,proto3" json:"selector_enclosure" validate:"selector"` unknownFields protoimpl.UnknownFields
CacheLifetime string `protobuf:"bytes,10,opt,name=cache_lifetime,json=cacheLifetime,proto3" json:"cache_lifetime"` sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
} }
func (x *Specs) Reset() { func (x *Specs) Reset() {
@ -165,20 +117,6 @@ func (x *Specs) GetSelectorCreated() string {
return "" return ""
} }
func (x *Specs) GetCreatedExtractFrom() ExtractFrom {
if x != nil {
return x.CreatedExtractFrom
}
return ExtractFrom_InnerText
}
func (x *Specs) GetCreatedAttributeName() string {
if x != nil {
return x.CreatedAttributeName
}
return ""
}
func (x *Specs) GetSelectorContent() string { func (x *Specs) GetSelectorContent() string {
if x != nil { if x != nil {
return x.SelectorContent return x.SelectorContent
@ -206,7 +144,7 @@ var file_proto_specs_proto_rawDesc = string([]byte{
0x0a, 0x11, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x73, 0x70, 0x65, 0x63, 0x73, 0x2e, 0x70, 0x72, 0x0a, 0x11, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x73, 0x70, 0x65, 0x63, 0x73, 0x2e, 0x70, 0x72,
0x6f, 0x74, 0x6f, 0x12, 0x0a, 0x72, 0x73, 0x73, 0x61, 0x6c, 0x63, 0x68, 0x65, 0x6d, 0x79, 0x1a, 0x6f, 0x74, 0x6f, 0x12, 0x0a, 0x72, 0x73, 0x73, 0x61, 0x6c, 0x63, 0x68, 0x65, 0x6d, 0x79, 0x1a,
0x13, 0x74, 0x61, 0x67, 0x67, 0x65, 0x72, 0x2f, 0x74, 0x61, 0x67, 0x67, 0x65, 0x72, 0x2e, 0x70, 0x13, 0x74, 0x61, 0x67, 0x67, 0x65, 0x72, 0x2f, 0x74, 0x61, 0x67, 0x67, 0x65, 0x72, 0x2e, 0x70,
0x72, 0x6f, 0x74, 0x6f, 0x22, 0xb6, 0x08, 0x0a, 0x05, 0x53, 0x70, 0x65, 0x63, 0x73, 0x12, 0x30, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0xef, 0x06, 0x0a, 0x05, 0x53, 0x70, 0x65, 0x63, 0x73, 0x12, 0x30,
0x0a, 0x03, 0x75, 0x72, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x42, 0x1e, 0x9a, 0x84, 0x9e, 0x0a, 0x03, 0x75, 0x72, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x42, 0x1e, 0x9a, 0x84, 0x9e,
0x03, 0x19, 0x6a, 0x73, 0x6f, 0x6e, 0x3a, 0x22, 0x75, 0x72, 0x6c, 0x22, 0x20, 0x76, 0x61, 0x6c, 0x03, 0x19, 0x6a, 0x73, 0x6f, 0x6e, 0x3a, 0x22, 0x75, 0x72, 0x6c, 0x22, 0x20, 0x76, 0x61, 0x6c,
0x69, 0x64, 0x61, 0x74, 0x65, 0x3a, 0x22, 0x75, 0x72, 0x6c, 0x22, 0x52, 0x03, 0x75, 0x72, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x3a, 0x22, 0x75, 0x72, 0x6c, 0x22, 0x52, 0x03, 0x75, 0x72, 0x6c,
@ -244,41 +182,26 @@ var file_proto_specs_proto_rawDesc = string([]byte{
0x6e, 0x3a, 0x22, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x5f, 0x63, 0x72, 0x65, 0x61, 0x6e, 0x3a, 0x22, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x5f, 0x63, 0x72, 0x65, 0x61,
0x74, 0x65, 0x64, 0x22, 0x20, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x3a, 0x22, 0x73, 0x74, 0x65, 0x64, 0x22, 0x20, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x3a, 0x22, 0x73,
0x65, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x22, 0x52, 0x0f, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x22, 0x52, 0x0f, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74,
0x6f, 0x72, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x12, 0x6b, 0x0a, 0x14, 0x63, 0x72, 0x65, 0x6f, 0x72, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x12, 0x65, 0x0a, 0x10, 0x73, 0x65, 0x6c,
0x61, 0x74, 0x65, 0x64, 0x5f, 0x65, 0x78, 0x74, 0x72, 0x61, 0x63, 0x74, 0x5f, 0x66, 0x72, 0x6f, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x18, 0x08, 0x20,
0x6d, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x17, 0x2e, 0x72, 0x73, 0x73, 0x61, 0x6c, 0x63, 0x01, 0x28, 0x09, 0x42, 0x3a, 0x9a, 0x84, 0x9e, 0x03, 0x35, 0x6a, 0x73, 0x6f, 0x6e, 0x3a, 0x22,
0x68, 0x65, 0x6d, 0x79, 0x2e, 0x45, 0x78, 0x74, 0x72, 0x61, 0x63, 0x74, 0x46, 0x72, 0x6f, 0x6d, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74,
0x42, 0x20, 0x9a, 0x84, 0x9e, 0x03, 0x1b, 0x6a, 0x73, 0x6f, 0x6e, 0x3a, 0x22, 0x63, 0x72, 0x65, 0x22, 0x20, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x3a, 0x22, 0x6f, 0x6d, 0x69, 0x74,
0x61, 0x74, 0x65, 0x64, 0x5f, 0x65, 0x78, 0x74, 0x72, 0x61, 0x63, 0x74, 0x5f, 0x66, 0x72, 0x6f, 0x65, 0x6d, 0x70, 0x74, 0x79, 0x2c, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x22, 0x52,
0x6d, 0x22, 0x52, 0x12, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x45, 0x78, 0x74, 0x72, 0x61, 0x0f, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74,
0x63, 0x74, 0x46, 0x72, 0x6f, 0x6d, 0x12, 0x58, 0x0a, 0x16, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x12, 0x61, 0x0a, 0x12, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x5f, 0x65, 0x6e, 0x63,
0x64, 0x5f, 0x61, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x6c, 0x6f, 0x73, 0x75, 0x72, 0x65, 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x42, 0x32, 0x9a, 0x84,
0x18, 0x0c, 0x20, 0x01, 0x28, 0x09, 0x42, 0x22, 0x9a, 0x84, 0x9e, 0x03, 0x1d, 0x6a, 0x73, 0x6f, 0x9e, 0x03, 0x2d, 0x6a, 0x73, 0x6f, 0x6e, 0x3a, 0x22, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x6f,
0x6e, 0x3a, 0x22, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x5f, 0x61, 0x74, 0x74, 0x72, 0x69, 0x72, 0x5f, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x73, 0x75, 0x72, 0x65, 0x22, 0x20, 0x76, 0x61, 0x6c,
0x62, 0x75, 0x74, 0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x22, 0x52, 0x14, 0x63, 0x72, 0x65, 0x61, 0x69, 0x64, 0x61, 0x74, 0x65, 0x3a, 0x22, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x22,
0x74, 0x65, 0x64, 0x41, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x52, 0x11, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x45, 0x6e, 0x63, 0x6c, 0x6f, 0x73,
0x12, 0x65, 0x0a, 0x10, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x5f, 0x63, 0x6f, 0x6e, 0x75, 0x72, 0x65, 0x12, 0x41, 0x0a, 0x0e, 0x63, 0x61, 0x63, 0x68, 0x65, 0x5f, 0x6c, 0x69, 0x66,
0x74, 0x65, 0x6e, 0x74, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x42, 0x3a, 0x9a, 0x84, 0x9e, 0x03, 0x65, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x42, 0x1a, 0x9a, 0x84, 0x9e,
0x35, 0x6a, 0x73, 0x6f, 0x6e, 0x3a, 0x22, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x5f, 0x03, 0x15, 0x6a, 0x73, 0x6f, 0x6e, 0x3a, 0x22, 0x63, 0x61, 0x63, 0x68, 0x65, 0x5f, 0x6c, 0x69,
0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x22, 0x20, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x66, 0x65, 0x74, 0x69, 0x6d, 0x65, 0x22, 0x52, 0x0d, 0x63, 0x61, 0x63, 0x68, 0x65, 0x4c, 0x69,
0x65, 0x3a, 0x22, 0x6f, 0x6d, 0x69, 0x74, 0x65, 0x6d, 0x70, 0x74, 0x79, 0x2c, 0x73, 0x65, 0x6c, 0x66, 0x65, 0x74, 0x69, 0x6d, 0x65, 0x42, 0x16, 0x5a, 0x14, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e,
0x65, 0x63, 0x74, 0x6f, 0x72, 0x22, 0x52, 0x0f, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x61, 0x6c, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x68, 0x74, 0x74, 0x70, 0x2f, 0x70, 0x62, 0x62, 0x06,
0x43, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x12, 0x61, 0x0a, 0x12, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
0x74, 0x6f, 0x72, 0x5f, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x73, 0x75, 0x72, 0x65, 0x18, 0x09, 0x20,
0x01, 0x28, 0x09, 0x42, 0x32, 0x9a, 0x84, 0x9e, 0x03, 0x2d, 0x6a, 0x73, 0x6f, 0x6e, 0x3a, 0x22,
0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x5f, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x73, 0x75,
0x72, 0x65, 0x22, 0x20, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x3a, 0x22, 0x73, 0x65,
0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x22, 0x52, 0x11, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x6f,
0x72, 0x45, 0x6e, 0x63, 0x6c, 0x6f, 0x73, 0x75, 0x72, 0x65, 0x12, 0x41, 0x0a, 0x0e, 0x63, 0x61,
0x63, 0x68, 0x65, 0x5f, 0x6c, 0x69, 0x66, 0x65, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x0a, 0x20, 0x01,
0x28, 0x09, 0x42, 0x1a, 0x9a, 0x84, 0x9e, 0x03, 0x15, 0x6a, 0x73, 0x6f, 0x6e, 0x3a, 0x22, 0x63,
0x61, 0x63, 0x68, 0x65, 0x5f, 0x6c, 0x69, 0x66, 0x65, 0x74, 0x69, 0x6d, 0x65, 0x22, 0x52, 0x0d,
0x63, 0x61, 0x63, 0x68, 0x65, 0x4c, 0x69, 0x66, 0x65, 0x74, 0x69, 0x6d, 0x65, 0x2a, 0x2b, 0x0a,
0x0b, 0x45, 0x78, 0x74, 0x72, 0x61, 0x63, 0x74, 0x46, 0x72, 0x6f, 0x6d, 0x12, 0x0d, 0x0a, 0x09,
0x49, 0x6e, 0x6e, 0x65, 0x72, 0x54, 0x65, 0x78, 0x74, 0x10, 0x00, 0x12, 0x0d, 0x0a, 0x09, 0x41,
0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, 0x10, 0x01, 0x42, 0x16, 0x5a, 0x14, 0x69, 0x6e,
0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x68, 0x74, 0x74, 0x70, 0x2f,
0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
}) })
var ( var (
@ -293,19 +216,16 @@ func file_proto_specs_proto_rawDescGZIP() []byte {
return file_proto_specs_proto_rawDescData return file_proto_specs_proto_rawDescData
} }
var file_proto_specs_proto_enumTypes = make([]protoimpl.EnumInfo, 1)
var file_proto_specs_proto_msgTypes = make([]protoimpl.MessageInfo, 1) var file_proto_specs_proto_msgTypes = make([]protoimpl.MessageInfo, 1)
var file_proto_specs_proto_goTypes = []any{ var file_proto_specs_proto_goTypes = []any{
(ExtractFrom)(0), // 0: rssalchemy.ExtractFrom (*Specs)(nil), // 0: rssalchemy.Specs
(*Specs)(nil), // 1: rssalchemy.Specs
} }
var file_proto_specs_proto_depIdxs = []int32{ var file_proto_specs_proto_depIdxs = []int32{
0, // 0: rssalchemy.Specs.created_extract_from:type_name -> rssalchemy.ExtractFrom 0, // [0:0] is the sub-list for method output_type
1, // [1:1] is the sub-list for method output_type 0, // [0:0] is the sub-list for method input_type
1, // [1:1] is the sub-list for method input_type 0, // [0:0] is the sub-list for extension type_name
1, // [1:1] is the sub-list for extension type_name 0, // [0:0] is the sub-list for extension extendee
1, // [1:1] is the sub-list for extension extendee 0, // [0:0] is the sub-list for field type_name
0, // [0:1] is the sub-list for field type_name
} }
func init() { file_proto_specs_proto_init() } func init() { file_proto_specs_proto_init() }
@ -318,14 +238,13 @@ func file_proto_specs_proto_init() {
File: protoimpl.DescBuilder{ File: protoimpl.DescBuilder{
GoPackagePath: reflect.TypeOf(x{}).PkgPath(), GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
RawDescriptor: unsafe.Slice(unsafe.StringData(file_proto_specs_proto_rawDesc), len(file_proto_specs_proto_rawDesc)), RawDescriptor: unsafe.Slice(unsafe.StringData(file_proto_specs_proto_rawDesc), len(file_proto_specs_proto_rawDesc)),
NumEnums: 1, NumEnums: 0,
NumMessages: 1, NumMessages: 1,
NumExtensions: 0, NumExtensions: 0,
NumServices: 0, NumServices: 0,
}, },
GoTypes: file_proto_specs_proto_goTypes, GoTypes: file_proto_specs_proto_goTypes,
DependencyIndexes: file_proto_specs_proto_depIdxs, DependencyIndexes: file_proto_specs_proto_depIdxs,
EnumInfos: file_proto_specs_proto_enumTypes,
MessageInfos: file_proto_specs_proto_msgTypes, MessageInfos: file_proto_specs_proto_msgTypes,
}.Build() }.Build()
File_proto_specs_proto = out.File File_proto_specs_proto = out.File

10
internal/extractors/pwextractor/pageparser.go
View File

@ -114,15 +114,7 @@ func (p *pageParser) extractPost(post playwright.Locator) (models.FeedItem, erro
item.Enclosure = newLocator(post, p.task.SelectorEnclosure).First().GetAttribute("src") item.Enclosure = newLocator(post, p.task.SelectorEnclosure).First().GetAttribute("src")
var createdDateStr string createdDateStr := newLocator(post, p.task.SelectorCreated).First().InnerText()
switch p.task.CreatedExtractFrom {
case models.ExtractFrom_InnerText:
createdDateStr = newLocator(post, p.task.SelectorCreated).First().InnerText()
case models.ExtractFrom_Attribute:
createdDateStr = newLocator(post, p.task.SelectorCreated).First().GetAttribute(p.task.CreatedAttributeName)
default:
return models.FeedItem{}, fmt.Errorf("invalid task.CreatedExtractFrom")
}
log.Debugf("date=%s", createdDateStr) log.Debugf("date=%s", createdDateStr)
createdDate, err := p.dateParser.ParseDate(createdDateStr) createdDate, err := p.dateParser.ParseDate(createdDateStr)
if err != nil { if err != nil {

31
internal/models/models.go
View File

@ -13,28 +13,19 @@ const (
TaskTypePageScreenshot = "page_screenshot" TaskTypePageScreenshot = "page_screenshot"
) )
type ExtractFrom int
const (
ExtractFrom_InnerText ExtractFrom = 0
ExtractFrom_Attribute ExtractFrom = 1
)
type Task struct { type Task struct {
// While adding new fields, dont forget to alter caching func // While adding new fields, dont forget to alter caching func
TaskType TaskType TaskType TaskType
URL string URL string
SelectorPost string SelectorPost string
SelectorTitle string SelectorTitle string
SelectorLink string SelectorLink string
SelectorDescription string SelectorDescription string
SelectorAuthor string SelectorAuthor string
SelectorCreated string SelectorCreated string
CreatedExtractFrom ExtractFrom SelectorContent string
CreatedAttributeName string SelectorEnclosure string
SelectorContent string Headers map[string]string
SelectorEnclosure string
Headers map[string]string
} }
func (t Task) CacheKey() string { func (t Task) CacheKey() string {

9
proto/specs.proto
View File

@ -6,11 +6,6 @@ import "tagger/tagger.proto";
option go_package = "internal/api/http/pb"; option go_package = "internal/api/http/pb";
enum ExtractFrom {
InnerText = 0;
Attribute = 1;
}
message Specs { message Specs {
string url = 1 [(tagger.tags) = "json:\"url\" validate:\"url\""]; string url = 1 [(tagger.tags) = "json:\"url\" validate:\"url\""];
string selector_post = 2 [(tagger.tags) = "json:\"selector_post\" validate:\"selector\""]; string selector_post = 2 [(tagger.tags) = "json:\"selector_post\" validate:\"selector\""];
@ -18,11 +13,7 @@ message Specs {
string selector_link = 4 [(tagger.tags) = "json:\"selector_link\" validate:\"selector\""]; string selector_link = 4 [(tagger.tags) = "json:\"selector_link\" validate:\"selector\""];
string selector_description = 5 [(tagger.tags) = "json:\"selector_description\" validate:\"omitempty,selector\""]; string selector_description = 5 [(tagger.tags) = "json:\"selector_description\" validate:\"omitempty,selector\""];
string selector_author = 6 [(tagger.tags) = "json:\"selector_author\" validate:\"selector\""]; string selector_author = 6 [(tagger.tags) = "json:\"selector_author\" validate:\"selector\""];
string selector_created = 7 [(tagger.tags) = "json:\"selector_created\" validate:\"selector\""]; string selector_created = 7 [(tagger.tags) = "json:\"selector_created\" validate:\"selector\""];
ExtractFrom created_extract_from = 11 [(tagger.tags) = "json:\"created_extract_from\""];
string created_attribute_name = 12 [(tagger.tags) = "json:\"created_attribute_name\""];
string selector_content = 8 [(tagger.tags) = "json:\"selector_content\" validate:\"omitempty,selector\""]; string selector_content = 8 [(tagger.tags) = "json:\"selector_content\" validate:\"omitempty,selector\""];
string selector_enclosure = 9 [(tagger.tags) = "json:\"selector_enclosure\" validate:\"selector\""]; string selector_enclosure = 9 [(tagger.tags) = "json:\"selector_enclosure\" validate:\"selector\""];
string cache_lifetime = 10 [(tagger.tags) = "json:\"cache_lifetime\""]; string cache_lifetime = 10 [(tagger.tags) = "json:\"cache_lifetime\""];