Not sure what’s up – but upgrading to latest (v2.0?) and asset downloading stopped working.
Our assets on our configured using GridFS as an asset store.
In the logs it just says “can’t find X thing in gridFS”.
Okay, will have a look. I am not sure if I even touched it.
Sorry, I cannot reproduce it, I also don’t know where the log entry should come from.
It will SHOW the assets – but when I try to download I get:
{
"logLevel": "Error",
"logLevel": "Error",
"message": "Caught and ignored exception: MongoDB.Driver.GridFS.GridFSFileNotFoundException with message: GridFS file not found: file id Index_7245467c-7544-42e3-935c-c126de91a560_0_. thrown from timer callback GrainTimer. TimerCallbackHandler:Squidex.Domain.Apps.Entities.Contents.Text.TextIndexerGrain->System.Threading.Tasks.Task <TryFlushAsync>b__25_0(System.Object)",
"message": "RunClientMessagePump has thrown exception",
"eventId": {
"eventId": {
"id": 101413
"id": 100326
},
},
"exception": {
"exception": {
"type": "System.OperationCanceledException",
"type": "MongoDB.Driver.GridFS.GridFSFileNotFoundException",
"message": "GridFS file not found: file id Index_7245467c-7544-42e3-935c-c126de91a560_0_.",
"message": "The operation was canceled.",
"stackTrace": " at System.Collections.Concurrent.BlockingCollection`1.TryTakeWithNoTimeValidation(T& item, Int32 millisecondsTimeout, CancellationToken cancellationToken, CancellationTokenSource combinedTokenSource)\n at System.Collections.Concurrent.BlockingCollection`1.TryTake(T& item, Int32 millisecondsTimeout, CancellationToken cancellationToken)\n at System.Collections.Concurrent.BlockingCollection`1.Take(CancellationToken cancellationToken)\n at Orleans.Runtime.HostedClient.RunClientMessagePump()"
"stackTrace": " at MongoDB.Driver.GridFS.GridFSBucket`1.DeleteAsync(TFileId id, CancellationToken cancellationToken)\n at Squidex.Infrastructure.Assets.MongoGridFsAssetStore.UploadFileCoreAsync(String id, Stream stream, Boolean overwrite, CancellationToken ct) in /src/src/Squidex.Infrastructure.MongoDb/Assets/MongoGridFsAssetStore.cs:line 122\n at Squidex.Domain.Apps.Entities.Contents.Text.PersistenceHelper.UploadDirectoryAsync(IAssetStore assetStore, DirectoryInfo directory, IndexCommit commit) in /src/src/Squidex.Domain.Apps.Entities/Contents/Text/PersistenceHelper.cs:line 55\n at Squidex.Domain.Apps.Entities.Contents.Text.TextIndexerGrain.FlushAsync() in /src/src/Squidex.Domain.Apps.Entities/Contents/Text/TextIndexerGrain.cs:line 191\n at Orleans.Runtime.GrainTimer.ForwardToAsyncCallback(Object state)"
},
},
"app": {
"app": {
"name": "Squidex",
"name": "Squidex",
"version": "1.0.0.0",
"version": "1.0.0.0",
"sessionId": "d4c48277-7621-41de-a64b-ad9a58415ac7"
"sessionId": "2bea2443-252a-4dbe-9de5-fcff871a35dd"
},
When I roll back to v1.16.x – it breaks all content – but assets work again.
This is helpful, lets see what I have changed in between.
I have pushed a potential fix.
Just updated to ‘latest’ tag. Same issue. Log dump:
},,
},,
"timestamp": "2019-04-02T16:33:18Z",,
"category": "Orleans.Runtime.Dispatcher",
},
"profiler": {,
"timestamp": "2019-04-02T16:32:51Z",,
"timestamp": "2019-04-02T15:08:53Z",,
"ExecuteResult": {,
,
{,
"elapsedMsTotal": 0,,
"logLevel": "Error",,
"elapsedMsAvg": 0,,
"message": "SelectTarget failed with Grain directory is stopping",,
"eventId": {,
"id": 100071,
},,
"exception": {,
"type": "System.InvalidOperationException",,
"category": "Orleans.Runtime.Dispatcher",
"category": "Orleans.Runtime.Dispatcher",
"count": 1,
},,
"message": "Grain directory is stopping",,
},
},
,
"mongoAssetRepository/FindAssetAsync": {,
"stackTrace": " at Orleans.Runtime.GrainDirectory.LocalGrainDirectory.CheckIfShouldForward(GrainId grainId, Int32 hopCount, String operationDescription)\n at Orleans.Runtime.GrainDirectory.LocalGrainDirectory.LookupAsync(GrainId grainId, Int32 hopCount)\n at Orleans.Runtime.Scheduler.AsyncClosureWorkItem`1.Execute()\n at Orleans.Runtime.Placement.RandomPlacementDirector.OnSelectActivation(PlacementStrategy strategy, GrainId target, IPlacementRuntime context)\n at Orleans.Runtime.Placement.PlacementDirectorsManager.SelectOrAddActivation(ActivationAddress sendingAddress, PlacementTarget targetGrain, IPlacementRuntime context, PlacementStrategy strategy)\n at Orleans.Runtime.Dispatcher.AddressMessageAsync(Message message, PlacementTarget target, PlacementStrategy strategy, ActivationAddress targetAddress)\n at Orleans.Runtime.Dispatcher.<>c__DisplayClass37_0.<<AsyncSendMessage>b__1>d.MoveNext()",
,
{,
{,
"elapsedMsTotal": 0,,
},,
"app": {,
"logLevel": "Error",,
"logLevel": "Error",,
"elapsedMsAvg": 0,,
"name": "Squidex",,
"count": 1,
"message": "SelectTarget failed with Grain directory is stopping",,
"message": "SelectTarget failed with Grain directory is stopping",,
"version": "1.0.0.0",,
"sessionId": "88933a0b-7567-42b2-bdc3-2e83fa3b1609",
},,
"eventId": {,
},,
"ExecuteAction": {,
"id": 100071,
"eventId": {,
"timestamp": "2019-04-02T16:33:19Z",,
"category": "Orleans.Runtime.Dispatcher",
"elapsedMsTotal": 0,,
},,
"id": 100071,
},,
},
"elapsedMsAvg": 0,,
"exception": {,
"type": "System.InvalidOperationException",,
"exception": {,
,
"count": 1,
},
"message": "Grain directory is stopping",,
"type": "System.InvalidOperationException",,
{,
"logLevel": "Warning",,
},,
"stackTrace": " at Orleans.Runtime.GrainDirectory.LocalGrainDirectory.CheckIfShouldForward(GrainId grainId, Int32 hopCount, String operationDescription)\n at Orleans.Runtime.GrainDirectory.LocalGrainDirectory.LookupAsync(GrainId grainId, Int32 hopCount)\n at Orleans.Runtime.Scheduler.AsyncClosureWorkItem`1.Execute()\n at Orleans.Runtime.Placement.RandomPlacementDirector.OnSelectActivation(PlacementStrategy strategy, GrainId target, IPlacementRuntime context)\n at Orleans.Runtime.Placement.PlacementDirectorsManager.SelectOrAddActivation(ActivationAddress sendingAddress, PlacementTarget targetGrain, IPlacementRuntime context, PlacementStrategy strategy)\n at Orleans.Runtime.Dispatcher.AddressMessageAsync(Message message, PlacementTarget target, PlacementStrategy strategy, ActivationAddress targetAddress)\n at Orleans.Runtime.Dispatcher.<>c__DisplayClass37_0.<<AsyncSendMessage>b__1>d.MoveNext()",
"message": "Grain directory is stopping",,
"stackTrace": " at Orleans.Runtime.GrainDirectory.LocalGrainDirectory.CheckIfShouldForward(GrainId grainId, Int32 hopCount, String operationDescription)\n at Orleans.Runtime.GrainDirectory.LocalGrainDirectory.LookupAsync(GrainId grainId, Int32 hopCount)\n at Orleans.Runtime.Scheduler.AsyncClosureWorkItem`1.Execute()\n at Orleans.Runtime.Placement.RandomPlacementDirector.OnSelectActivation(PlacementStrategy strategy, GrainId target, IPlacementRuntime context)\n at Orleans.Runtime.Placement.PlacementDirectorsManager.SelectOrAddActivation(ActivationAddress sendingAddress, PlacementTarget targetGrain, IPlacementRuntime context, PlacementStrategy strategy)\n at Orleans.Runtime.Dispatcher.AddressMessageAsync(Message message, PlacementTarget target, PlacementStrategy strategy, ActivationAddress targetAddress)\n at Orleans.Runtime.Dispatcher.<>c__DisplayClass37_0.<<AsyncSendMessage>b__1>d.MoveNext()",
"message": "Discarding OneWay rejection for message OneWay S10.0.0.8:11111:291918777*cli/4eddee9f@d5dd969d->*grn/FFE8F7B3/00000000 #155: . Exc = Grain directory is stopping",,
"filters": {,
},,
"app": {,
},,
"eventId": {,
"costs": 0.5,
},,
"name": "Squidex",,
"app": {,
"id": 101002,
},,
"elapsedRequestMs": 1,,
"version": "1.0.0.0",,
"name": "Squidex",,
"version": "1.0.0.0",,
"app": {,
"app": {,
"sessionId": "d4c48277-7621-41de-a64b-ad9a58415ac7",
},,
"sessionId": "2bea2443-252a-4dbe-9de5-fcff871a35dd",
"name": "Squidex",,
"name": "Squidex",,
"version": "1.0.0.0",,
"timestamp": "2019-04-02T16:32:51Z",,
},,
"version": "1.0.0.0",,
"sessionId": "88933a0b-7567-42b2-bdc3-2e83fa3b1609",
"sessionId": "2a420207-67ff-4946-b46d-c1a3de063805",
"category": "Orleans.Runtime.Dispatcher",
"timestamp": "2019-04-02T15:08:54Z",,
"category": "Orleans.Runtime.Dispatcher",
},,
},,
},
,
},
"timestamp": "2019-04-02T16:33:19Z",,
"web": {,
"requestId": "b04538a5-a463-460c-9d4e-210c159ab57f",,
{,
,
"category": "Orleans.Runtime.Dispatcher",
},
"requestPath": "/api/assets/5f0ec056-33e8-4358-a8f7-178f16db7686",,
"logLevel": "Warning",,
{,
"logLevel": "Warning",,
,
"requestMethod": "GET",
"message": "Discarding OneWay rejection for message OneWay S10.0.0.7:11111:291913740*cli/88c5d8a2@5e6c7cb0->*grn/68072C12/00000000+10.0.0.7:11111@291913740 #52527: . Exc = Grain directory is stopping",,
"eventId": {,
"message": "Discarding OneWay rejection for message OneWay S10.0.0.6:11111:291854905*cli/3565e899@805ec8b8->*grn/FFE8F7B3/00000000 #610433: . Exc = Grain directory is stopping",,
{,
},,
"timestamp": "2019-04-02T16:33:52Z",
"id": 101002,
"eventId": {,
"logLevel": "Error",,
"message": "Directory.RegisterAsync S10.0.0.8:11111:291918777*cli/4eddee9f@1c0c3a00 failed.",,
},
},,
"id": 101002,
},,
"eventId": {,
,
"app": {,
"name": "Squidex",,
"app": {,
"id": 101317,
{,
"name": "Squidex",,
"logLevel": "Fatal",,
},,
"version": "1.0.0.0",,
"sessionId": "d4c48277-7621-41de-a64b-ad9a58415ac7",
"message": "Failed to send result.",,
"exception": {,
"type": "System.InvalidOperationException",,
},,
"eventId": {,
"id": 99,
"message": "Grain directory is stopping",,
"timestamp": "2019-04-02T16:32:51Z",,
"category": "Orleans.Runtime.Dispatcher",
},,
"stackTrace": " at Orleans.Runtime.GrainDirectory.LocalGrainDirectory.CheckIfShouldForward(GrainId grainId, Int32 hopCount, String operationDescription)\n at Orleans.Runtime.GrainDirectory.LocalGrainDirectory.RegisterAsync(ActivationAddress address, Boolean singleActivation, Int32 hopCount)\n at Orleans.OrleansTaskExtentions.LogException(Task task, ILogger logger, ErrorCode errorCode, String message)",
},,
},
"exception": {,
"type": "Squidex.Infrastructure.Assets.AssetNotFoundException",,
"app": {,
,
{,
"message": "An asset with name 'Id=25cf2ac4-0577-4778-96fc-52ff0dd51ec2, Version=0' does not exist.",,
"name": "Squidex",,
"version": "1.0.0.0",,
"logLevel": "Error",,
"stackTrace": " at Squidex.Infrastructure.Assets.MongoGridFsAssetStore.DownloadAsync(String id, Int64 version, String suffix, Stream stream, CancellationToken ct) in /src/src/Squidex.Infrastructure.MongoDb/Assets/MongoGridFsAssetStore.cs:line 77\n at Squidex.Areas.Api.Controllers.Assets.AssetContentController.<>c__DisplayClass4_0.<<GetAssetContent>b__0>d.MoveNext() in /src/src/Squidex/Areas/Api/Controllers/Assets/AssetContentController.cs:line 131\n--- End of stack trace from previous location where exception was thrown ---\n at Squidex.Web.Pipeline.FileCallbackResultExecutor.ExecuteAsync(ActionContext context, FileCallbackResult result) in /src/src/Squidex.Web/Pipeline/FileCallbackResultExecutor.cs:line 29",
},,
"sessionId": "88933a0b-7567-42b2-bdc3-2e83fa3b1609",
"message": "Directory.RegisterAsync S10.0.0.7:11111:291913740*cli/88c5d8a2@a4cbd28b failed.",,
"eventId": {,
"app": {,
},,
"timestamp": "2019-04-02T16:33:20Z",,
"id": 101317,
"name": "Squidex",,
"version": "1.0.0.0",,
"category": "Orleans.Runtime.ClientObserverRegistrar",
},,
"exception": {,
"sessionId": "2a420207-67ff-4946-b46d-c1a3de063805",
},
,
"type": "System.InvalidOperationException",,
},,
"web": {,
{,
"message": "Grain directory is stopping",,
"stackTrace": " at Orleans.Runtime.GrainDirectory.LocalGrainDirectory.CheckIfShouldForward(GrainId grainId, Int32 hopCount, String operationDescription)\n at Orleans.Runtime.GrainDirectory.LocalGrainDirectory.RegisterAsync(ActivationAddress address, Boolean singleActivation, Int32 hopCount)\n at Orleans.OrleansTaskExtentions.LogException(Task task, ILogger logger, ErrorCode errorCode, String message)",
"requestId": "950ab15d-9cc0-4838-94e8-abb06b8095a8",,
"logLevel": "Error",,
"message": "OnClientRefreshTimer has thrown an exceptions.",,
},,
"requestPath": "/assets/25cf2ac4-0577-4778-96fc-52ff0dd51ec2",,
"requestMethod": "GET",,
"eventId": {,
"app": {,
"version": "1.0.0.0",,
"name": "Squidex",,
"routeValues": {,
"id": 101319,
},,
"version": "1.0.0.0",,
"area": "Api",,
"action": "GetAssetContent",,
"exception": {,
"sessionId": "d4c48277-7621-41de-a64b-ad9a58415ac7",
"sessionId": "2bea2443-252a-4dbe-9de5-fcff871a35dd",
},,
"controller": "AssetContent",
"type": "System.InvalidOperationException",,
},,
"timestamp": "2019-04-02T16:32:51Z",,
"timestamp": "2019-04-02T15:08:54Z",,
},
"message": "Grain directory is stopping",,
"stackTrace": " at Orleans.Runtime.GrainDirectory.LocalGrainDirectory.CheckIfShouldForward(GrainId grainId, Int32 hopCount, String operationDescription)\n at Orleans.Runtime.GrainDirectory.LocalGrainDirectory.RegisterAsync(ActivationAddress address, Boolean singleActivation, Int32 hopCount)\n at Orleans.OrleansTaskExtentions.LogException(Task task, ILogger logger, ErrorCode errorCode, String message)\n at Orleans.Runtime.ClientObserverRegistrar.OnClientRefreshTimer(Object data)",
"category": "Orleans.Runtime.ClientObserverRegistrar",
"category": "Orleans.Runtime.Dispatcher",
},,
"timestamp": "2019-04-02T16:33:54Z",,
},,
},
},
,
"category": "Microsoft.AspNetCore.Mvc.Infrastructure.VirtualFileResultExecutor",
"app": {,
,
{,
{,
},
"name": "Squidex",,
"version": "1.0.0.0",,
"logLevel": "Error",,
"logLevel": "Error",,
,
{,
"sessionId": "88933a0b-7567-42b2-bdc3-2e83fa3b1609",
"message": "OnClientRefreshTimer has thrown an exceptions.",,
"message": "Directory.RegisterAsync S10.0.0.6:11111:291854905*cli/3565e899@8893f994 failed.",,
"eventId": {,
"logLevel": "Information",,
},,
"eventId": {,
"id": 101319,
"id": 101317,
"profiler": {,
"timestamp": "2019-04-02T16:33:20Z",,
"category": "Orleans.Runtime.ClientObserverRegistrar",
},,
},,
"ExecuteResult": {,
"elapsedMsTotal": 139,,
},
"exception": {,
"exception": {,
"type": "System.InvalidOperationException",,
"elapsedMsAvg": 139,,
"count": 1,
,
"type": "System.InvalidOperationException",,
"message": "Grain directory is stopping",,
"stackTrace": " at Orleans.Runtime.GrainDirectory.LocalGrainDirectory.CheckIfShouldForward(GrainId grainId, Int32 hopCount, String operationDescription)\n at Orleans.Runtime.GrainDirectory.LocalGrainDirectory.RegisterAsync(ActivationAddress address, Boolean singleActivation, Int32 hopCount)\n at Orleans.OrleansTaskExtentions.LogException(Task task, ILogger logger, ErrorCode errorCode, String message)",
},,
{,
"message": "Grain directory is stopping",,
"stackTrace": " at Orleans.Runtime.GrainDirectory.LocalGrainDirectory.CheckIfShouldForward(GrainId grainId, Int32 hopCount, String operationDescription)\n at Orleans.Runtime.GrainDirectory.LocalGrainDirectory.RegisterAsync(ActivationAddress address, Boolean singleActivation, Int32 hopCount)\n at Orleans.OrleansTaskExtentions.LogException(Task task, ILogger logger, ErrorCode errorCode, String message)\n at Orleans.Runtime.ClientObserverRegistrar.OnClientRefreshTimer(Object data)",
},,
"mongoAssetRepository/FindAssetAsync": {,
"logLevel": "Error",,
"elapsedMsTotal": 1,,
},,
"app": {,
"name": "Squidex",,
"elapsedMsAvg": 1,,
"app": {,
"name": "Squidex",,
"version": "1.0.0.0",,
"count": 1,
},,
"message": "RunClientMessagePump has thrown exception",,
"version": "1.0.0.0",,
"sessionId": "2bea2443-252a-4dbe-9de5-fcff871a35dd",
},,
"ExecuteAction": {,
"timestamp": "2019-04-02T15:08:54Z",,
"category": "Orleans.Runtime.ClientObserverRegistrar",
"eventId": {,
"sessionId": "d4c48277-7621-41de-a64b-ad9a58415ac7",
},,
},
"elapsedMsTotal": 1,,
"id": 100326,
},,
"timestamp": "2019-04-02T16:32:51Z",,
,
"elapsedMsAvg": 1,,
"count": 1,
"exception": {,
"category": "Orleans.Runtime.ClientObserverRegistrar",
{,
"logLevel": "Error",,
},
"type": "System.OperationCanceledException",,
},
"message": "OnClientRefreshTimer has thrown an exceptions.",,
},,
"message": "The operation was canceled.",,
,
{,
"eventId": {,
"filters": {,
"stackTrace": " at System.Collections.Concurrent.BlockingCollection`1.TryTakeWithNoTimeValidation(T& item, Int32 millisecondsTimeout, CancellationToken cancellationToken, CancellationTokenSource combinedTokenSource)\n at System.Collections.Concurrent.BlockingCollection`1.TryTake(T& item, Int32 millisecondsTimeout, CancellationToken cancellationToken)\n at System.Collections.Concurrent.BlockingCollection`1.Take(CancellationToken cancellationToken)\n at Orleans.Runtime.HostedClient.RunClientMessagePump()",
},,
"logLevel": "Error",,
"id": 101319,
"costs": 0.5,
},,
"app": {,
"message": "RunClientMessagePump has thrown exception",,
},,
"exception": {,
"elapsedRequestMs": 142,,
"name": "Squidex",,
"eventId": {,
"id": 100326,
"type": "System.InvalidOperationException",,
"app": {,
"version": "1.0.0.0",,
"sessionId": "88933a0b-7567-42b2-bdc3-2e83fa3b1609",
},,
"message": "Grain directory is stopping",,
"name": "Squidex",,
"version": "1.0.0.0",,
},,
"exception": {,
"stackTrace": " at Orleans.Runtime.GrainDirectory.LocalGrainDirectory.CheckIfShouldForward(GrainId grainId, Int32 hopCount, String operationDescription)\n at Orleans.Runtime.GrainDirectory.LocalGrainDirectory.RegisterAsync(ActivationAddress address, Boolean singleActivation, Int32 hopCount)\n at Orleans.OrleansTaskExtentions.LogException(Task task, ILogger logger, ErrorCode errorCode, String message)\n at Orleans.Runtime.ClientObserverRegistrar.OnClientRefreshTimer(Object data)",
},,
"sessionId": "2a420207-67ff-4946-b46d-c1a3de063805",
"timestamp": "2019-04-02T16:33:20Z",,
"type": "System.OperationCanceledException",,
"message": "The operation was canceled.",,
"app": {,
},,
"category": "Orleans.Runtime.HostedClient",
},
"stackTrace": " at System.Collections.Concurrent.BlockingCollection`1.TryTakeWithNoTimeValidation(T& item, Int32 millisecondsTimeout, CancellationToken cancellationToken, CancellationTokenSource combinedTokenSource)\n at System.Collections.Concurrent.BlockingCollection`1.TryTake(T& item, Int32 millisecondsTimeout, CancellationToken cancellationToken)\n at System.Collections.Concurrent.BlockingCollection`1.Take(CancellationToken cancellationToken)\n at Orleans.Runtime.HostedClient.RunClientMessagePump()",
"name": "Squidex",,
"web": {,
"requestId": "950ab15d-9cc0-4838-94e8-abb06b8095a8",,
,
},,
"version": "1.0.0.0",,
"sessionId": "2bea2443-252a-4dbe-9de5-fcff871a35dd",
"requestPath": "/api/assets/25cf2ac4-0577-4778-96fc-52ff0dd51ec2",,
"app": {,
"name": "Squidex",,
},,
"requestMethod": "GET",
},,
"version": "1.0.0.0",,
"timestamp": "2019-04-02T15:08:54Z",,
"category": "Orleans.Runtime.ClientObserverRegistrar",
"timestamp": "2019-04-02T16:33:54Z",
"sessionId": "d4c48277-7621-41de-a64b-ad9a58415ac7",
},,
},
},
,
"timestamp": "2019-04-02T16:32:51Z",,
,
{,
"category": "Orleans.Runtime.HostedClient",
},
"logLevel": "Error",,
"message": "RunClientMessagePump has thrown exception",,
,
"eventId": {,
"id": 100326,
},,
"exception": {,
"type": "System.OperationCanceledException",,
"message": "The operation was canceled.",,
"stackTrace": " at System.Collections.Concurrent.BlockingCollection`1.TryTakeWithNoTimeValidation(T& item, Int32 millisecondsTimeout, CancellationToken cancellationToken, CancellationTokenSource combinedTokenSource)\n at System.Collections.Concurrent.BlockingCollection`1.TryTake(T& item, Int32 millisecondsTimeout, CancellationToken cancellationToken)\n at System.Collections.Concurrent.BlockingCollection`1.Take(CancellationToken cancellationToken)\n at Orleans.Runtime.HostedClient.RunClientMessagePump()",
},,
"app": {,
"name": "Squidex",,
"version": "1.0.0.0",,
"sessionId": "2bea2443-252a-4dbe-9de5-fcff871a35dd",
},,
"timestamp": "2019-04-02T15:08:54Z",,
"category": "Orleans.Runtime.HostedClient",
},
,```Can you backup your database including the assets and send me the backup?
Or have a look how the assets are named in your fs.files Collection.
My bad, @Sebastian – I updated to “:latest” not “:dev” – assets work properly now on “dev”.
Basically it’s fixed.
Perfect, thank you for your help and patience.