POFO Patches

src/base/DataType.php

@@ -53,7 +53,7 @@ public function setupPaginationUrl($array, $feed): void
 
         // if the feed provides a root relative URL, make it whole again based on the feed.
         if ($url && UrlHelper::isRootRelativeUrl($url)) {
-            $url = UrlHelper::hostInfo($feed->feedUrl) . $url;
+            $url = UrlHelper::hostInfo($feed->getFeedUrl()) . $url;
         }
 
         // Replace the mapping value with the actual URL

src/base/Field.php

@@ -8,6 +8,7 @@
 use craft\base\Component;
 use craft\errors\ElementNotFoundException;
 use craft\feedme\helpers\DataHelper;
+use craft\feedme\models\FeedModel;
 use craft\feedme\Plugin;
 use craft\helpers\Json;
 use Throwable;
@@ -46,7 +47,7 @@ abstract class Field extends Component
     public mixed $field = null;
 
     /**
-     * @var
+     * @var FeedModel|array
      */
     public mixed $feed = null;
 
@@ -167,7 +168,7 @@ protected function populateElementFields($elementIds, $nodeKey = null): void
 
                 // Arrayed content doesn't provide defaults because it's unable to determine how many items it _should_ return
                 // This also checks if there was any data that corresponds on the same array index/level as our element
-                $value = Hash::get($fieldValue, $nodeKey, $default);
+                $value = Hash::get($fieldValue, $nodeKey ?? 0, $default);
 
                 if ($value) {
                     $fieldData[$elementId][$fieldHandle] = $value;
@@ -176,9 +177,16 @@ protected function populateElementFields($elementIds, $nodeKey = null): void
         }
 
         // Now, for each element, we need to save the contents
-        foreach ($fieldData as $elementId => $fieldContent) {
+        foreach ($fieldData as $elementId => $elementData) {
             $element = $elementsService->getElementById($elementId, null, Hash::get($this->feed, 'siteId'));
 
+            $fieldHandles = collect($element->getFieldLayout()->getCustomFields())->pluck('handle')->all();
+            $attributeKeys = array_diff(array_keys($elementData), $fieldHandles);
+
+            $attributeContent = collect($elementData)->only($attributeKeys)->all();
+            $element->setAttributes($attributeContent);
+
+            $fieldContent = collect($elementData)->except($attributeKeys)->all();
             $element->setFieldValues($fieldContent);
 
             Plugin::debug([
@@ -191,7 +199,7 @@ protected function populateElementFields($elementIds, $nodeKey = null): void
                 Plugin::error('`{handle}` - Unable to save sub-field: `{e}`.', ['e' => Json::encode($element->getErrors()), 'handle' => $this->fieldHandle]);
             }
 
-            Plugin::info('`{handle}` - Processed {name} [`#{id}`]({url}) sub-fields with content: `{content}`.', ['name' => $element::displayName(), 'id' => $elementId, 'url' => $element->cpEditUrl, 'handle' => $this->fieldHandle, 'content' => Json::encode($fieldContent)]);
+            Plugin::info('`{handle}` - Processed {name} [`#{id}`]({url}) sub-fields with content: `{content}`.', ['name' => $element::displayName(), 'id' => $elementId, 'url' => $element->cpEditUrl, 'handle' => $this->fieldHandle, 'content' => Json::encode($elementData)]);
         }
     }
 

src/console/controllers/FeedsController.php

@@ -49,7 +49,11 @@ public function options($actionID): array
         $options[] = 'limit';
         $options[] = 'offset';
         $options[] = 'continueOnError';
-        $options[] = 'all';
+
+        if ($actionID !== 'queue') {
+            $options[] = 'all';
+        }
+
         return $options;
     }
 
@@ -117,4 +121,30 @@ protected function queueFeed($feed, $limit = null, $offset = null, bool $continu
 
         $this->stdout('done' . PHP_EOL, Console::FG_GREEN);
     }
+
+    /**
+     * Execute a feed without the queue
+     */
+    public function actionExecute(string $feedId=null): int
+    {
+        $queue = new \yii\queue\sync\Queue;
+        Plugin::getInstance()->queue = $queue;
+
+        $config = [
+            'feed' => Plugin::getInstance()?->getFeeds()->getFeedById($feedId),
+            'limit' => $this->limit,
+            'offset' => $this->offset,
+            'continueOnError' => $this->continueOnError,
+        ];
+
+        // Push the first job in to the queue
+        $job = new FeedImport($config);
+        $queue->push($job);
+
+        // Run the queue, subsequent pages will be pushed in at the
+        // end of an existing page
+        $queue->run();
+
+        return 1;
+    }
 }

src/controllers/FeedsController.php

@@ -39,7 +39,7 @@ class FeedsController extends Controller
      */
     public function actionFeedsIndex(): Response
     {
-        $variables['feeds'] = Plugin::$plugin->feeds->getFeeds();
+        $variables['feeds'] = Plugin::$plugin->feeds->getFeeds('name asc');
 
         return $this->renderTemplate('feed-me/feeds/index', $variables);
     }

src/elements/Entry.php

@@ -124,9 +124,10 @@ public function getQuery($settings, array $params = []): mixed
      */
     public function setModel($settings): ElementInterface
     {
-        $this->element = new EntryElement();
-        $this->element->sectionId = $settings['elementGroup'][EntryElement::class]['section'];
-        $this->element->typeId = $settings['elementGroup'][EntryElement::class]['entryType'];
+        $this->element = new EntryElement([
+            'sectionId' => $settings['elementGroup'][EntryElement::class]['section'],
+            'typeId' => $settings['elementGroup'][EntryElement::class]['entryType'],
+        ]);
 
         $section = Craft::$app->getSections()->getSectionById($this->element->sectionId);
         $siteId = Hash::get($settings, 'siteId');

src/fields/Matrix.php

@@ -170,13 +170,10 @@ public function parseField(): mixed
         $index = 1;
         $resultBlocks = [];
         foreach ($expanded as $blockData) {
-            // all the fields are empty and setEmptyValues is off, ignore the block
-            if (
-                !empty(array_filter(
-                    $blockData['fields'],
-                    fn($value) => (is_string($value) && !empty($value)) || (is_array($value) && !empty(array_filter($value)))
-                ))
-            ) {
+            $setEmptyValues = $this->feed['setEmptyValues'] ?? false;
+            $isNotEmpty = collect($blockData['fields'])->filter(fn ($value) => ! empty($value))->isNotEmpty();
+
+            if ($setEmptyValues || (! $setEmptyValues && $isNotEmpty)) {
                 $resultBlocks['new' . $index++] = $blockData;
             }
         }

src/helpers/AssetHelper.php

@@ -2,6 +2,7 @@
 
 namespace craft\feedme\helpers;
 
+use Aws\S3\Exception\S3Exception;
 use Cake\Utility\Hash;
 use Craft;
 use craft\base\ElementInterface;
@@ -100,39 +101,60 @@ public static function fetchRemoteImage(array $urls, $fieldInfo, $feed, $field =
         // Download each image. Note we've already checked if there's an existing asset and if the
         // user has set to use that instead, so we're good to proceed.
         foreach ($urls as $url) {
-            try {
-                $filename = $newFilename ? AssetsHelper::prepareAssetName($newFilename, false) : self::getRemoteUrlFilename($url);
-
-                $fetchedImage = $tempFeedMePath . $filename;
-
-                // But also check if we've downloaded this recently, use the copy in the temp directory
-                $cachedImage = FileHelper::findFiles($tempFeedMePath, [
-                    'only' => [$filename],
-                    'recursive' => false,
-                ]);
-
-                Plugin::info('Fetching remote image `{i}` - `{j}`', ['i' => $url, 'j' => $filename]);
-
-                if (!$cachedImage) {
-                    self::downloadFile($url, $fetchedImage);
-                } else {
-                    $fetchedImage = $cachedImage[0];
+            $try = 0;
+            $maxTries = 5;
+            while($try++ < $maxTries) {
+                try {
+                    $filename = $newFilename ? AssetsHelper::prepareAssetName($newFilename, false) : self::getRemoteUrlFilename($url);
+
+                    $fetchedImage = $tempFeedMePath . $filename;
+
+                    // But also check if we've downloaded this recently, use the copy in the temp directory
+                    $cachedImage = FileHelper::findFiles($tempFeedMePath, [
+                        'only' => [$filename],
+                        'recursive' => false,
+                    ]);
+
+                    Plugin::info('Fetching remote image `{i}` - `{j}`', ['i' => $url, 'j' => $filename]);
+
+                    if (!$cachedImage) {
+                        self::downloadFile($url, $fetchedImage);
+                    } else {
+                        $fetchedImage = $cachedImage[0];
+                    }
+
+                    $result = self::createAsset($fetchedImage, $filename, $folderId, $field, $element, $conflict, Hash::get($feed, 'updateSearchIndexes'));
+
+                    if ($result) {
+                        $uploadedAssets[] = $result;
+                    } else {
+                        Plugin::error('Failed to create asset from `{i}`', ['i' => $url]);
+                    }
+
+                    break;
+                } catch (Throwable $e) {
+                    if ($try < $maxTries) {
+                        $prev = $e;
+                        while ($prev) {
+                            if (get_class($prev) === \Aws\S3\Exception\S3Exception::class && $prev->getStatusCode() === 400) {
+                                Plugin::info('`{handle}` - Asset error, resetting credentials. {e}.', ['e' => $prev->getMessage(), 'handle' => $field->handle]);
+                                Craft::$app->set('volumes', ["class" => "craft\services\Volumes"]);
+                                Craft::$app->set('fs', ["class" => "craft\services\Fs"]);
+                            }
+                            $prev = $prev->getPrevious();
+                        }
+                        Plugin::info('`{handle}` - Asset error, trying again: `{url}` - `{e}`.', ['url' => $url, 'e' => $e->getMessage(), 'handle' => $field->handle]);
+                        sleep($try);
+                        continue;
+                    }
+
+                    if ($field) {
+                        Plugin::error('`{handle}` - Asset error: `{url}` - `{e}` `{t}`.', ['url' => $url, 'e' => $e->getMessage(), 'handle' => $field->handle, 't' => $e->getTraceAsString()]);
+                    } else {
+                        Plugin::error('Asset error: `{url}` - `{e}`.', ['url' => $url, 'e' => $e->getMessage()]);
+                    }
+                    Craft::$app->getErrorHandler()->logException($e);
                 }
-
-                $result = self::createAsset($fetchedImage, $filename, $folderId, $field, $element, $conflict, Hash::get($feed, 'updateSearchIndexes'));
-
-                if ($result) {
-                    $uploadedAssets[] = $result;
-                } else {
-                    Plugin::error('Failed to create asset from `{i}`', ['i' => $url]);
-                }
-            } catch (Throwable $e) {
-                if ($field) {
-                    Plugin::error('`{handle}` - Asset error: `{url}` - `{e}`.', ['url' => $url, 'e' => $e->getMessage(), 'handle' => $field->handle]);
-                } else {
-                    Plugin::error('Asset error: `{url}` - `{e}`.', ['url' => $url, 'e' => $e->getMessage()]);
-                }
-                Craft::$app->getErrorHandler()->logException($e);
             }
         }
 

src/models/FeedModel.php

@@ -10,6 +10,7 @@
 use craft\feedme\base\ElementInterface;
 use craft\feedme\helpers\DuplicateHelper;
 use craft\feedme\Plugin;
+use craft\helpers\App;
 use DateTime;
 
 /**
@@ -156,6 +157,11 @@ public function __toString()
         return Craft::t('feed-me', $this->name);
     }
 
+    public function getFeedUrl(): string
+    {
+        return App::parseEnv($this->feedUrl);
+    }
+
     /**
      * @return string
      */
@@ -234,10 +240,12 @@ public function getFeedMapping(bool $usePrimaryElement = true): mixed
     public function getNextPagination(): bool
     {
         if (!$this->paginationUrl || !filter_var($this->paginationUrl, FILTER_VALIDATE_URL)) {
+            Plugin::info('No paginationUrl, stopping.');
             return false;
         }
 
         // Set the URL dynamically on the feed, then kick off processing again
+        Plugin::info('Next paginationUrl found. Creating new Queue job with '.$this->feedUrl);
         $this->feedUrl = $this->paginationUrl;
 
         return true;

src/queue/jobs/FeedImport.php

@@ -87,6 +87,7 @@ public function execute($queue): void
             }
 
             $feedSettings = Plugin::$plugin->process->beforeProcessFeed($this->feed, $feedData);
+            Plugin::info('Processing `' . $this->feed->getFeedUrl() . '`');
 
             $feedData = $feedSettings['feedData'];
 

src/services/DataTypes.php

@@ -204,27 +204,43 @@ public function getRawData($url, $feedId = null): array
             ]);
         }
 
-        try {
-            $client = Plugin::$plugin->service->createGuzzleClient($feedId);
-            $options = Plugin::$plugin->service->getRequestOptions($feedId);
+        $tries = 0;
+        $maxTries = 10;
+        while (++$tries <= $maxTries) {
+            try {
+                $client = Plugin::$plugin->service->createGuzzleClient($feedId);
+                $options = Plugin::$plugin->service->getRequestOptions($feedId);
 
-            $resp = $client->request('GET', $url, $options);
-            $data = (string)$resp->getBody();
+                $resp = $client->request('GET', $url, $options);
+                $data = (string)$resp->getBody();
 
-            // Save headers for later
-            $this->_headers = $resp->getHeaders();
+                // Save headers for later
+                $this->_headers = $resp->getHeaders();
 
-            $response = ['success' => true, 'data' => $data];
-        } catch (Exception $e) {
-            $response = ['success' => false, 'error' => $e->getMessage()];
-            Craft::$app->getErrorHandler()->logException($e);
-        }
+                $response = ['success' => true, 'data' => $data];
 
-        return $this->_triggerEventAfterFetchFeed([
-            'url' => $url,
-            'feedId' => $feedId,
-            'response' => $response,
-        ]);
+                return $this->_triggerEventAfterFetchFeed([
+                    'url' => $url,
+                    'feedId' => $feedId,
+                    'response' => $response,
+                ]);
+            } catch (Exception $e) {
+                $response = ['success' => false, 'error' => $e->getMessage()];
+                Craft::$app->getErrorHandler()->logException($e);
+
+                if ($tries === $maxTries) {
+                    return $this->_triggerEventAfterFetchFeed([
+                        'url' => $url,
+                        'feedId' => $feedId,
+                        'response' => $response,
+                    ]);
+                }
+                else {
+                    Plugin::info('HTTP error #' . $tries . ' at ' . $url . ', ' . $e->getMessage());
+                    sleep(($tries ^ 2) / 2); // wait up to 40.5 seconds on the 9th try
+                }
+            }
+        }
     }
 
     /**
@@ -234,10 +250,10 @@ public function getRawData($url, $feedId = null): array
      */
     public function getFeedData($feedModel, bool $usePrimaryElement = true): mixed
     {
-        $feedDataResponse = $feedModel->getDataType()->getFeed($feedModel->feedUrl, $feedModel, $usePrimaryElement);
+        $feedDataResponse = $feedModel->getDataType()->getFeed($feedModel->getFeedUrl(), $feedModel, $usePrimaryElement);
 
         $event = new FeedDataEvent([
-            'url' => $feedModel->feedUrl,
+            'url' => $feedModel->getFeedUrl(),
             'response' => $feedDataResponse,
             'feedId' => $feedModel->id,
         ]);

src/templates/_includes/elements/assets/map.html

@@ -42,6 +42,13 @@
         type: 'select',
         options: folders,
     },
+}, {
+    name: 'Alternative Text',
+    handle: 'alt',
+    instructions: 'Add descriptive text to describe the appearance and function of an image on a page.'|t('feed-me'),
+    default: {
+        type: 'text',
+    },
 }, {
     name: 'Asset ID',
     handle: 'id',

src/web/twig/variables/FeedMeVariable.php

@@ -240,7 +240,11 @@ public function getElementLayoutByField($type, $field): ?array
         }
 
         if (($fieldLayout = Craft::$app->getFields()->getLayoutById($source->fieldLayoutId)) !== null) {
-            return $fieldLayout->getCustomFields();
+            $customFields = $fieldLayout->getCustomFields();
+            if ($field instanceof \craft\fields\Assets) {
+                array_unshift($customFields, new \craft\fields\PlainText(['name' => 'Alt', 'handle' => 'alt']));
+            }
+            return $customFields;
         }
 
         return null;