Compare commits

..

No commits in common. "0e32cc3f176baa3314e08b9eaf6a565e0b61f0fe" and "da86d3b5d2d6e6b2ab6a92d02618b908ad5a3a9e" have entirely different histories.

15 changed files with 115 additions and 227 deletions

View File

@ -1,5 +0,0 @@
export type EnumValue = {
label: string
value: number
}
export type Enum = EnumValue[]

View File

@ -1,6 +1,6 @@
<script setup lang="ts"> <script setup lang="ts">
import TextField from "@/components/inputs/TextField.vue"; import TextField from "@/components/TextField.vue";
import Btn from "@/components/Btn.vue"; import Btn from "@/components/Btn.vue";
import {onMounted, onUnmounted, ref, watch} from "vue"; import {onMounted, onUnmounted, ref, watch} from "vue";
import Modal from "@/components/Modal.vue"; import Modal from "@/components/Modal.vue";

View File

@ -0,0 +1,45 @@
<script setup lang="ts">
import type {Field} from "@/urlmaker/specs.ts";
import {getCurrentInstance, onMounted, useTemplateRef} from "vue";
const {field, focused} = defineProps<{
field: Field,
focused?: boolean,
}>();
const id = 'field' + getCurrentInstance()?.uid;
const model = defineModel();
const inputRef = useTemplateRef('field');
onMounted(() => {
if(focused) inputRef.value?.focus();
})
</script>
<template>
<div class="field">
<div class="label"><label :for="id">{{ field.label }}</label></div>
<div class="input">
<input :type="field.input_type" :name="field.name" :id="id" v-model="model" ref="field"/>
</div>
</div>
</template>
<style scoped lang="scss">
div.field {
margin: 0 0 8px 0;
}
div.label {
font-size: 0.9em;
}
div.input {
margin: 2px 0 0 0;
box-sizing: border-box;
input {
box-sizing: border-box;
width: 100%;
padding: 2px;
}
}
</style>

View File

@ -1,19 +1,12 @@
<script setup lang="ts"> <script setup lang="ts">
import {fields, InputType, type SpecField} from '@/urlmaker/specs.ts'; import {fields, InputType} from '@/urlmaker/specs.ts';
import TextField from "@/components/inputs/TextField.vue"; import TextField from "@/components/TextField.vue";
import RadioButtons from "@/components/inputs/RadioButtons.vue";
import {useWizardStore} from "@/stores/wizard.ts"; import {useWizardStore} from "@/stores/wizard.ts";
import {groupBy} from "es-toolkit";
const store = useWizardStore(); const store = useWizardStore();
const groups: SpecField[][] = []; const groups = groupBy(fields, item => item.group || '');
for (const field of fields) {
if(groups.length === 0 || groups[groups.length - 1][0].group != field.group) {
groups.push([field]);
} else {
groups[groups.length - 1].push(field);
}
}
</script> </script>
@ -23,22 +16,12 @@ for (const field of fields) {
<template v-for="field in group"> <template v-for="field in group">
<TextField <TextField
v-if="field.input_type === InputType.Url || field.input_type === InputType.Text" v-if="field.input_type === InputType.Url || field.input_type === InputType.Text"
v-show="!field.show_if || field.show_if(store.specs)"
:name="field.name" :name="field.name"
:label="field.label" :label="field.label"
:input_type="field.input_type" :input_type="field.input_type"
:model-value="store.specs[field.name]" :model-value="store.specs[field.name]"
@update:model-value="event => store.updateSpec(field.name, event)" @update:model-value="event => store.updateSpec(field.name, event)"
></TextField> ></TextField>
<RadioButtons
v-if="field.input_type === InputType.Radio"
v-show="!field.show_if || field.show_if(store.specs)"
:name="field.name"
:label="field.label"
:values="field.enum!"
:model-value="store.specs[field.name]"
@update:model-value="event => store.updateSpec(field.name, event)"
></RadioButtons>
</template> </template>
</div> </div>
</div> </div>
@ -46,6 +29,6 @@ for (const field of fields) {
<style scoped lang="scss"> <style scoped lang="scss">
div.group { div.group {
margin: 20px 0; margin: 6px 0;
} }
</style> </style>

View File

@ -1,51 +0,0 @@
<script setup lang="ts">
import {getCurrentInstance} from "vue";
import type {Enum} from "@/common/enum.ts";
const {name, label, values} = defineProps<{
name: string
label: string,
values: Enum,
}>();
const componentId = 'field' + getCurrentInstance()?.uid;
const model = defineModel();
</script>
<template>
<div class="field">
<span class="field-label"><label>{{ label }}</label></span>
<template class="value" v-for="enumValue in values">
<input
type="radio"
:name="name"
:value="enumValue.value"
:id="`${componentId}_${enumValue.value}`"
v-model="model"
/>
<label class="radio-label" :for="`${componentId}_${enumValue.value}`">{{ enumValue.label }}</label>
</template>
</div>
</template>
<style scoped lang="scss">
div.field {
margin: 0 0 8px 0;
}
.field-label {
font-size: 0.9em;
margin-right: 8px;
}
.radio-label {
font-size: 0.9em;
}
input, .radio-label, .field-label {
vertical-align: middle;
}
input {
margin-top: 0;
}
</style>

View File

@ -1,5 +1,5 @@
import {defineStore} from "pinia"; import {defineStore} from "pinia";
import {emptySpecs, type SpecField, fields, type Specs, type SpecValue} from "@/urlmaker/specs.ts"; import {emptySpecs, type SpecField, fields, type Specs} from "@/urlmaker/specs.ts";
import {computed, reactive} from "vue"; import {computed, reactive} from "vue";
import {debounce} from "es-toolkit"; import {debounce} from "es-toolkit";
@ -22,7 +22,7 @@ export const useWizardStore = defineStore('wizard', () => {
localStorage.setItem(LOCAL_STORAGE_KEY, JSON.stringify(specs)); localStorage.setItem(LOCAL_STORAGE_KEY, JSON.stringify(specs));
}, 100); }, 100);
function updateSpec(fieldName: keyof Specs, newValue: SpecValue) { function updateSpec(fieldName: keyof Specs, newValue: string) {
specs[fieldName] = newValue; specs[fieldName] = newValue;
updateLocalStorage(); updateLocalStorage();
} }

View File

@ -20,7 +20,6 @@ export namespace rssalchemy {
selector_author?: string; selector_author?: string;
selector_created?: string; selector_created?: string;
created_extract_from?: ExtractFrom; created_extract_from?: ExtractFrom;
created_attribute_name?: string;
selector_content?: string; selector_content?: string;
selector_enclosure?: string; selector_enclosure?: string;
cache_lifetime?: string; cache_lifetime?: string;
@ -52,9 +51,6 @@ export namespace rssalchemy {
if ("created_extract_from" in data && data.created_extract_from != undefined) { if ("created_extract_from" in data && data.created_extract_from != undefined) {
this.created_extract_from = data.created_extract_from; this.created_extract_from = data.created_extract_from;
} }
if ("created_attribute_name" in data && data.created_attribute_name != undefined) {
this.created_attribute_name = data.created_attribute_name;
}
if ("selector_content" in data && data.selector_content != undefined) { if ("selector_content" in data && data.selector_content != undefined) {
this.selector_content = data.selector_content; this.selector_content = data.selector_content;
} }
@ -114,12 +110,6 @@ export namespace rssalchemy {
set created_extract_from(value: ExtractFrom) { set created_extract_from(value: ExtractFrom) {
pb_1.Message.setField(this, 11, value); pb_1.Message.setField(this, 11, value);
} }
get created_attribute_name() {
return pb_1.Message.getFieldWithDefault(this, 12, "") as string;
}
set created_attribute_name(value: string) {
pb_1.Message.setField(this, 12, value);
}
get selector_content() { get selector_content() {
return pb_1.Message.getFieldWithDefault(this, 8, "") as string; return pb_1.Message.getFieldWithDefault(this, 8, "") as string;
} }
@ -147,7 +137,6 @@ export namespace rssalchemy {
selector_author?: string; selector_author?: string;
selector_created?: string; selector_created?: string;
created_extract_from?: ExtractFrom; created_extract_from?: ExtractFrom;
created_attribute_name?: string;
selector_content?: string; selector_content?: string;
selector_enclosure?: string; selector_enclosure?: string;
cache_lifetime?: string; cache_lifetime?: string;
@ -177,9 +166,6 @@ export namespace rssalchemy {
if (data.created_extract_from != null) { if (data.created_extract_from != null) {
message.created_extract_from = data.created_extract_from; message.created_extract_from = data.created_extract_from;
} }
if (data.created_attribute_name != null) {
message.created_attribute_name = data.created_attribute_name;
}
if (data.selector_content != null) { if (data.selector_content != null) {
message.selector_content = data.selector_content; message.selector_content = data.selector_content;
} }
@ -201,7 +187,6 @@ export namespace rssalchemy {
selector_author?: string; selector_author?: string;
selector_created?: string; selector_created?: string;
created_extract_from?: ExtractFrom; created_extract_from?: ExtractFrom;
created_attribute_name?: string;
selector_content?: string; selector_content?: string;
selector_enclosure?: string; selector_enclosure?: string;
cache_lifetime?: string; cache_lifetime?: string;
@ -230,9 +215,6 @@ export namespace rssalchemy {
if (this.created_extract_from != null) { if (this.created_extract_from != null) {
data.created_extract_from = this.created_extract_from; data.created_extract_from = this.created_extract_from;
} }
if (this.created_attribute_name != null) {
data.created_attribute_name = this.created_attribute_name;
}
if (this.selector_content != null) { if (this.selector_content != null) {
data.selector_content = this.selector_content; data.selector_content = this.selector_content;
} }
@ -264,8 +246,6 @@ export namespace rssalchemy {
writer.writeString(7, this.selector_created); writer.writeString(7, this.selector_created);
if (this.created_extract_from != ExtractFrom.InnerText) if (this.created_extract_from != ExtractFrom.InnerText)
writer.writeEnum(11, this.created_extract_from); writer.writeEnum(11, this.created_extract_from);
if (this.created_attribute_name.length)
writer.writeString(12, this.created_attribute_name);
if (this.selector_content.length) if (this.selector_content.length)
writer.writeString(8, this.selector_content); writer.writeString(8, this.selector_content);
if (this.selector_enclosure.length) if (this.selector_enclosure.length)
@ -305,9 +285,6 @@ export namespace rssalchemy {
case 11: case 11:
message.created_extract_from = reader.readEnum(); message.created_extract_from = reader.readEnum();
break; break;
case 12:
message.created_attribute_name = reader.readString();
break;
case 8: case 8:
message.selector_content = reader.readString(); message.selector_content = reader.readString();
break; break;

View File

@ -1,12 +1,10 @@
import { import {
validateAttribute,
validateDuration, validateDuration,
validateSelector, validateSelector,
validateUrl, validateUrl,
type validator type validator
} from "@/urlmaker/validators.ts"; } from "@/urlmaker/validators.ts";
import {rssalchemy} from "@/urlmaker/proto/specs.ts"; import {rssalchemy} from "@/urlmaker/proto/specs.ts";
import type {Enum} from "@/common/enum.ts";
export type SpecKey = ReturnType<rssalchemy.Specs['toObject']>; export type SpecKey = ReturnType<rssalchemy.Specs['toObject']>;
export type SpecValue = string | number; export type SpecValue = string | number;
@ -21,7 +19,7 @@ export enum InputType {
export interface SpecField { export interface SpecField {
name: keyof Specs name: keyof Specs
input_type: InputType input_type: InputType
enum?: Enum, enum_values?: {[k: number]: string}
label: string label: string
default: SpecValue default: SpecValue
validate: validator validate: validator
@ -86,26 +84,13 @@ export const fields: SpecField[] = [
{ {
name: 'created_extract_from', name: 'created_extract_from',
input_type: InputType.Radio, input_type: InputType.Radio,
enum: [ enum_values: rssalchemy.ExtractFrom,
{label: 'Inner Text', value: rssalchemy.ExtractFrom.InnerText},
{label: 'Attribute', value: rssalchemy.ExtractFrom.Attribute},
],
label: 'Extract from', label: 'Extract from',
default: rssalchemy.ExtractFrom.InnerText, default: rssalchemy.ExtractFrom.InnerText,
validate: value => Object.values(rssalchemy.ExtractFrom).includes(value), validate: value => Object.values(rssalchemy.ExtractFrom).includes(value),
group: 'created', group: 'created',
show_if: specs => !!specs.selector_created, show_if: specs => !!specs.selector_created,
}, },
{
name: 'created_attribute_name',
input_type: InputType.Text,
label: 'Attribute name',
default: '',
validate: validateAttribute,
show_if: specs =>
!!specs.selector_created && specs.created_extract_from === rssalchemy.ExtractFrom.Attribute,
group: 'created',
},
{ {
name: 'selector_content', name: 'selector_content',

View File

@ -26,10 +26,6 @@ export function validateSelector(s: SpecValue): boolean {
} }
} }
export function validateAttribute(s: SpecValue): boolean {
return /([^\t\n\f \/>"'=]+)/.test(s as string);
}
export function validateDuration(s: SpecValue): boolean { export function validateDuration(s: SpecValue): boolean {
return /^\d+[smh]$/.test(s as string); return /^\d+[smh]$/.test(s as string);
} }

View File

@ -75,28 +75,18 @@ func (h *Handler) handleRender(c echo.Context) error {
return echo.NewHTTPError(400, fmt.Errorf("decode specs: %w", err)) return echo.NewHTTPError(400, fmt.Errorf("decode specs: %w", err))
} }
extractFrom, ok := map[pb.ExtractFrom]models.ExtractFrom{
pb.ExtractFrom_InnerText: models.ExtractFrom_InnerText,
pb.ExtractFrom_Attribute: models.ExtractFrom_Attribute,
}[specs.CreatedExtractFrom]
if !ok {
return echo.NewHTTPError(400, "invalid extract from")
}
task := models.Task{ task := models.Task{
TaskType: models.TaskTypeExtract, TaskType: models.TaskTypeExtract,
URL: specs.Url, URL: specs.Url,
SelectorPost: specs.SelectorPost, SelectorPost: specs.SelectorPost,
SelectorTitle: specs.SelectorTitle, SelectorTitle: specs.SelectorTitle,
SelectorLink: specs.SelectorLink, SelectorLink: specs.SelectorLink,
SelectorDescription: specs.SelectorDescription, SelectorDescription: specs.SelectorDescription,
SelectorAuthor: specs.SelectorAuthor, SelectorAuthor: specs.SelectorAuthor,
SelectorCreated: specs.SelectorCreated, SelectorCreated: specs.SelectorCreated,
CreatedExtractFrom: extractFrom, SelectorContent: specs.SelectorContent,
CreatedAttributeName: specs.CreatedAttributeName, SelectorEnclosure: specs.SelectorEnclosure,
SelectorContent: specs.SelectorContent, Headers: extractHeaders(c),
SelectorEnclosure: specs.SelectorEnclosure,
Headers: extractHeaders(c),
} }
cacheLifetime, err := time.ParseDuration(specs.CacheLifetime) cacheLifetime, err := time.ParseDuration(specs.CacheLifetime)

View File

@ -69,21 +69,20 @@ func (ExtractFrom) EnumDescriptor() ([]byte, []int) {
} }
type Specs struct { type Specs struct {
state protoimpl.MessageState `protogen:"open.v1"` state protoimpl.MessageState `protogen:"open.v1"`
Url string `protobuf:"bytes,1,opt,name=url,proto3" json:"url" validate:"url"` Url string `protobuf:"bytes,1,opt,name=url,proto3" json:"url" validate:"url"`
SelectorPost string `protobuf:"bytes,2,opt,name=selector_post,json=selectorPost,proto3" json:"selector_post" validate:"selector"` SelectorPost string `protobuf:"bytes,2,opt,name=selector_post,json=selectorPost,proto3" json:"selector_post" validate:"selector"`
SelectorTitle string `protobuf:"bytes,3,opt,name=selector_title,json=selectorTitle,proto3" json:"selector_title" validate:"selector"` SelectorTitle string `protobuf:"bytes,3,opt,name=selector_title,json=selectorTitle,proto3" json:"selector_title" validate:"selector"`
SelectorLink string `protobuf:"bytes,4,opt,name=selector_link,json=selectorLink,proto3" json:"selector_link" validate:"selector"` SelectorLink string `protobuf:"bytes,4,opt,name=selector_link,json=selectorLink,proto3" json:"selector_link" validate:"selector"`
SelectorDescription string `protobuf:"bytes,5,opt,name=selector_description,json=selectorDescription,proto3" json:"selector_description" validate:"omitempty,selector"` SelectorDescription string `protobuf:"bytes,5,opt,name=selector_description,json=selectorDescription,proto3" json:"selector_description" validate:"omitempty,selector"`
SelectorAuthor string `protobuf:"bytes,6,opt,name=selector_author,json=selectorAuthor,proto3" json:"selector_author" validate:"selector"` SelectorAuthor string `protobuf:"bytes,6,opt,name=selector_author,json=selectorAuthor,proto3" json:"selector_author" validate:"selector"`
SelectorCreated string `protobuf:"bytes,7,opt,name=selector_created,json=selectorCreated,proto3" json:"selector_created" validate:"selector"` SelectorCreated string `protobuf:"bytes,7,opt,name=selector_created,json=selectorCreated,proto3" json:"selector_created" validate:"selector"`
CreatedExtractFrom ExtractFrom `protobuf:"varint,11,opt,name=created_extract_from,json=createdExtractFrom,proto3,enum=rssalchemy.ExtractFrom" json:"created_extract_from"` CreatedExtractFrom ExtractFrom `protobuf:"varint,11,opt,name=created_extract_from,json=createdExtractFrom,proto3,enum=rssalchemy.ExtractFrom" json:"created_extract_from"`
CreatedAttributeName string `protobuf:"bytes,12,opt,name=created_attribute_name,json=createdAttributeName,proto3" json:"created_attribute_name"` SelectorContent string `protobuf:"bytes,8,opt,name=selector_content,json=selectorContent,proto3" json:"selector_content" validate:"omitempty,selector"`
SelectorContent string `protobuf:"bytes,8,opt,name=selector_content,json=selectorContent,proto3" json:"selector_content" validate:"omitempty,selector"` SelectorEnclosure string `protobuf:"bytes,9,opt,name=selector_enclosure,json=selectorEnclosure,proto3" json:"selector_enclosure" validate:"selector"`
SelectorEnclosure string `protobuf:"bytes,9,opt,name=selector_enclosure,json=selectorEnclosure,proto3" json:"selector_enclosure" validate:"selector"` CacheLifetime string `protobuf:"bytes,10,opt,name=cache_lifetime,json=cacheLifetime,proto3" json:"cache_lifetime"`
CacheLifetime string `protobuf:"bytes,10,opt,name=cache_lifetime,json=cacheLifetime,proto3" json:"cache_lifetime"` unknownFields protoimpl.UnknownFields
unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache
sizeCache protoimpl.SizeCache
} }
func (x *Specs) Reset() { func (x *Specs) Reset() {
@ -172,13 +171,6 @@ func (x *Specs) GetCreatedExtractFrom() ExtractFrom {
return ExtractFrom_InnerText return ExtractFrom_InnerText
} }
func (x *Specs) GetCreatedAttributeName() string {
if x != nil {
return x.CreatedAttributeName
}
return ""
}
func (x *Specs) GetSelectorContent() string { func (x *Specs) GetSelectorContent() string {
if x != nil { if x != nil {
return x.SelectorContent return x.SelectorContent
@ -206,7 +198,7 @@ var file_proto_specs_proto_rawDesc = string([]byte{
0x0a, 0x11, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x73, 0x70, 0x65, 0x63, 0x73, 0x2e, 0x70, 0x72, 0x0a, 0x11, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x73, 0x70, 0x65, 0x63, 0x73, 0x2e, 0x70, 0x72,
0x6f, 0x74, 0x6f, 0x12, 0x0a, 0x72, 0x73, 0x73, 0x61, 0x6c, 0x63, 0x68, 0x65, 0x6d, 0x79, 0x1a, 0x6f, 0x74, 0x6f, 0x12, 0x0a, 0x72, 0x73, 0x73, 0x61, 0x6c, 0x63, 0x68, 0x65, 0x6d, 0x79, 0x1a,
0x13, 0x74, 0x61, 0x67, 0x67, 0x65, 0x72, 0x2f, 0x74, 0x61, 0x67, 0x67, 0x65, 0x72, 0x2e, 0x70, 0x13, 0x74, 0x61, 0x67, 0x67, 0x65, 0x72, 0x2f, 0x74, 0x61, 0x67, 0x67, 0x65, 0x72, 0x2e, 0x70,
0x72, 0x6f, 0x74, 0x6f, 0x22, 0xb6, 0x08, 0x0a, 0x05, 0x53, 0x70, 0x65, 0x63, 0x73, 0x12, 0x30, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0xdc, 0x07, 0x0a, 0x05, 0x53, 0x70, 0x65, 0x63, 0x73, 0x12, 0x30,
0x0a, 0x03, 0x75, 0x72, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x42, 0x1e, 0x9a, 0x84, 0x9e, 0x0a, 0x03, 0x75, 0x72, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x42, 0x1e, 0x9a, 0x84, 0x9e,
0x03, 0x19, 0x6a, 0x73, 0x6f, 0x6e, 0x3a, 0x22, 0x75, 0x72, 0x6c, 0x22, 0x20, 0x76, 0x61, 0x6c, 0x03, 0x19, 0x6a, 0x73, 0x6f, 0x6e, 0x3a, 0x22, 0x75, 0x72, 0x6c, 0x22, 0x20, 0x76, 0x61, 0x6c,
0x69, 0x64, 0x61, 0x74, 0x65, 0x3a, 0x22, 0x75, 0x72, 0x6c, 0x22, 0x52, 0x03, 0x75, 0x72, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x3a, 0x22, 0x75, 0x72, 0x6c, 0x22, 0x52, 0x03, 0x75, 0x72, 0x6c,
@ -251,34 +243,28 @@ var file_proto_specs_proto_rawDesc = string([]byte{
0x42, 0x20, 0x9a, 0x84, 0x9e, 0x03, 0x1b, 0x6a, 0x73, 0x6f, 0x6e, 0x3a, 0x22, 0x63, 0x72, 0x65, 0x42, 0x20, 0x9a, 0x84, 0x9e, 0x03, 0x1b, 0x6a, 0x73, 0x6f, 0x6e, 0x3a, 0x22, 0x63, 0x72, 0x65,
0x61, 0x74, 0x65, 0x64, 0x5f, 0x65, 0x78, 0x74, 0x72, 0x61, 0x63, 0x74, 0x5f, 0x66, 0x72, 0x6f, 0x61, 0x74, 0x65, 0x64, 0x5f, 0x65, 0x78, 0x74, 0x72, 0x61, 0x63, 0x74, 0x5f, 0x66, 0x72, 0x6f,
0x6d, 0x22, 0x52, 0x12, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x45, 0x78, 0x74, 0x72, 0x61, 0x6d, 0x22, 0x52, 0x12, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x45, 0x78, 0x74, 0x72, 0x61,
0x63, 0x74, 0x46, 0x72, 0x6f, 0x6d, 0x12, 0x58, 0x0a, 0x16, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x63, 0x74, 0x46, 0x72, 0x6f, 0x6d, 0x12, 0x65, 0x0a, 0x10, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74,
0x64, 0x5f, 0x61, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x6f, 0x72, 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09,
0x18, 0x0c, 0x20, 0x01, 0x28, 0x09, 0x42, 0x22, 0x9a, 0x84, 0x9e, 0x03, 0x1d, 0x6a, 0x73, 0x6f, 0x42, 0x3a, 0x9a, 0x84, 0x9e, 0x03, 0x35, 0x6a, 0x73, 0x6f, 0x6e, 0x3a, 0x22, 0x73, 0x65, 0x6c,
0x6e, 0x3a, 0x22, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x5f, 0x61, 0x74, 0x74, 0x72, 0x69, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x22, 0x20, 0x76,
0x62, 0x75, 0x74, 0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x22, 0x52, 0x14, 0x63, 0x72, 0x65, 0x61, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x3a, 0x22, 0x6f, 0x6d, 0x69, 0x74, 0x65, 0x6d, 0x70,
0x74, 0x65, 0x64, 0x41, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x74, 0x79, 0x2c, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x22, 0x52, 0x0f, 0x73, 0x65,
0x12, 0x65, 0x0a, 0x10, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x5f, 0x63, 0x6f, 0x6e, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x12, 0x61, 0x0a,
0x74, 0x65, 0x6e, 0x74, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x42, 0x3a, 0x9a, 0x84, 0x9e, 0x03, 0x12, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x5f, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x73,
0x35, 0x6a, 0x73, 0x6f, 0x6e, 0x3a, 0x22, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x5f, 0x75, 0x72, 0x65, 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x42, 0x32, 0x9a, 0x84, 0x9e, 0x03, 0x2d,
0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x22, 0x20, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x6a, 0x73, 0x6f, 0x6e, 0x3a, 0x22, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x5f, 0x65,
0x65, 0x3a, 0x22, 0x6f, 0x6d, 0x69, 0x74, 0x65, 0x6d, 0x70, 0x74, 0x79, 0x2c, 0x73, 0x65, 0x6c, 0x6e, 0x63, 0x6c, 0x6f, 0x73, 0x75, 0x72, 0x65, 0x22, 0x20, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x61,
0x65, 0x63, 0x74, 0x6f, 0x72, 0x22, 0x52, 0x0f, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x74, 0x65, 0x3a, 0x22, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x22, 0x52, 0x11, 0x73,
0x43, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x12, 0x61, 0x0a, 0x12, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x45, 0x6e, 0x63, 0x6c, 0x6f, 0x73, 0x75, 0x72, 0x65,
0x74, 0x6f, 0x72, 0x5f, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x73, 0x75, 0x72, 0x65, 0x18, 0x09, 0x20, 0x12, 0x41, 0x0a, 0x0e, 0x63, 0x61, 0x63, 0x68, 0x65, 0x5f, 0x6c, 0x69, 0x66, 0x65, 0x74, 0x69,
0x01, 0x28, 0x09, 0x42, 0x32, 0x9a, 0x84, 0x9e, 0x03, 0x2d, 0x6a, 0x73, 0x6f, 0x6e, 0x3a, 0x22, 0x6d, 0x65, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x42, 0x1a, 0x9a, 0x84, 0x9e, 0x03, 0x15, 0x6a,
0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x5f, 0x65, 0x6e, 0x63, 0x6c, 0x6f, 0x73, 0x75, 0x73, 0x6f, 0x6e, 0x3a, 0x22, 0x63, 0x61, 0x63, 0x68, 0x65, 0x5f, 0x6c, 0x69, 0x66, 0x65, 0x74,
0x72, 0x65, 0x22, 0x20, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x3a, 0x22, 0x73, 0x65, 0x69, 0x6d, 0x65, 0x22, 0x52, 0x0d, 0x63, 0x61, 0x63, 0x68, 0x65, 0x4c, 0x69, 0x66, 0x65, 0x74,
0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x22, 0x52, 0x11, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x69, 0x6d, 0x65, 0x2a, 0x2b, 0x0a, 0x0b, 0x45, 0x78, 0x74, 0x72, 0x61, 0x63, 0x74, 0x46, 0x72,
0x72, 0x45, 0x6e, 0x63, 0x6c, 0x6f, 0x73, 0x75, 0x72, 0x65, 0x12, 0x41, 0x0a, 0x0e, 0x63, 0x61, 0x6f, 0x6d, 0x12, 0x0d, 0x0a, 0x09, 0x49, 0x6e, 0x6e, 0x65, 0x72, 0x54, 0x65, 0x78, 0x74, 0x10,
0x63, 0x68, 0x65, 0x5f, 0x6c, 0x69, 0x66, 0x65, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x0a, 0x20, 0x01, 0x00, 0x12, 0x0d, 0x0a, 0x09, 0x41, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, 0x10, 0x01,
0x28, 0x09, 0x42, 0x1a, 0x9a, 0x84, 0x9e, 0x03, 0x15, 0x6a, 0x73, 0x6f, 0x6e, 0x3a, 0x22, 0x63, 0x42, 0x16, 0x5a, 0x14, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2f, 0x61, 0x70, 0x69,
0x61, 0x63, 0x68, 0x65, 0x5f, 0x6c, 0x69, 0x66, 0x65, 0x74, 0x69, 0x6d, 0x65, 0x22, 0x52, 0x0d, 0x2f, 0x68, 0x74, 0x74, 0x70, 0x2f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
0x63, 0x61, 0x63, 0x68, 0x65, 0x4c, 0x69, 0x66, 0x65, 0x74, 0x69, 0x6d, 0x65, 0x2a, 0x2b, 0x0a,
0x0b, 0x45, 0x78, 0x74, 0x72, 0x61, 0x63, 0x74, 0x46, 0x72, 0x6f, 0x6d, 0x12, 0x0d, 0x0a, 0x09,
0x49, 0x6e, 0x6e, 0x65, 0x72, 0x54, 0x65, 0x78, 0x74, 0x10, 0x00, 0x12, 0x0d, 0x0a, 0x09, 0x41,
0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, 0x10, 0x01, 0x42, 0x16, 0x5a, 0x14, 0x69, 0x6e,
0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x68, 0x74, 0x74, 0x70, 0x2f,
0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
}) })
var ( var (

View File

@ -114,15 +114,7 @@ func (p *pageParser) extractPost(post playwright.Locator) (models.FeedItem, erro
item.Enclosure = newLocator(post, p.task.SelectorEnclosure).First().GetAttribute("src") item.Enclosure = newLocator(post, p.task.SelectorEnclosure).First().GetAttribute("src")
var createdDateStr string createdDateStr := newLocator(post, p.task.SelectorCreated).First().InnerText()
switch p.task.CreatedExtractFrom {
case models.ExtractFrom_InnerText:
createdDateStr = newLocator(post, p.task.SelectorCreated).First().InnerText()
case models.ExtractFrom_Attribute:
createdDateStr = newLocator(post, p.task.SelectorCreated).First().GetAttribute(p.task.CreatedAttributeName)
default:
return models.FeedItem{}, fmt.Errorf("invalid task.CreatedExtractFrom")
}
log.Debugf("date=%s", createdDateStr) log.Debugf("date=%s", createdDateStr)
createdDate, err := p.dateParser.ParseDate(createdDateStr) createdDate, err := p.dateParser.ParseDate(createdDateStr)
if err != nil { if err != nil {

View File

@ -13,28 +13,19 @@ const (
TaskTypePageScreenshot = "page_screenshot" TaskTypePageScreenshot = "page_screenshot"
) )
type ExtractFrom int
const (
ExtractFrom_InnerText ExtractFrom = 0
ExtractFrom_Attribute ExtractFrom = 1
)
type Task struct { type Task struct {
// While adding new fields, dont forget to alter caching func // While adding new fields, dont forget to alter caching func
TaskType TaskType TaskType TaskType
URL string URL string
SelectorPost string SelectorPost string
SelectorTitle string SelectorTitle string
SelectorLink string SelectorLink string
SelectorDescription string SelectorDescription string
SelectorAuthor string SelectorAuthor string
SelectorCreated string SelectorCreated string
CreatedExtractFrom ExtractFrom SelectorContent string
CreatedAttributeName string SelectorEnclosure string
SelectorContent string Headers map[string]string
SelectorEnclosure string
Headers map[string]string
} }
func (t Task) CacheKey() string { func (t Task) CacheKey() string {

View File

@ -21,7 +21,6 @@ message Specs {
string selector_created = 7 [(tagger.tags) = "json:\"selector_created\" validate:\"selector\""]; string selector_created = 7 [(tagger.tags) = "json:\"selector_created\" validate:\"selector\""];
ExtractFrom created_extract_from = 11 [(tagger.tags) = "json:\"created_extract_from\""]; ExtractFrom created_extract_from = 11 [(tagger.tags) = "json:\"created_extract_from\""];
string created_attribute_name = 12 [(tagger.tags) = "json:\"created_attribute_name\""];
string selector_content = 8 [(tagger.tags) = "json:\"selector_content\" validate:\"omitempty,selector\""]; string selector_content = 8 [(tagger.tags) = "json:\"selector_content\" validate:\"omitempty,selector\""];
string selector_enclosure = 9 [(tagger.tags) = "json:\"selector_enclosure\" validate:\"selector\""]; string selector_enclosure = 9 [(tagger.tags) = "json:\"selector_enclosure\" validate:\"selector\""];