我有一个返回数据库查询结果的函数。这些结果非常大，我现在希望将它们作为流传递，以便客户端可以更快地开始处理它们，并且内存使用率更低。但我真的不知道如何做到这一点，以下功能有效，但我想知道如何更改它以便在从第一个表读取时开始流式传输。
// Generated on 2015-06-21 using generator-angular 0.11.1
'use strict';
// # Globbing
// for performance reasons we're only matching one level down:
// 'test/spec/{,*/}*.js'
// use this if you want to recursively match all subfolders:
// 'test/spec/**/*.js'
module.exports = function (grunt) {
// Load grunt tasks automatically
require('load-grunt-tasks')(grunt);
// Time how long tasks take. Can help when optimizing build times
require('time-grunt')(grunt);
// Configurable paths for the application
var appConfig = {
app: require('./bower.json').appPath || 'app',
dist: 'dist'
};
// Define the configuration for all the tasks
grunt.initConfig({
// Project settings
yeoman: appConfig,
// Watches files for changes and runs tasks based on the changed files
watch: {
bower: {
files: ['bower.json'],
tasks: ['wiredep']
},
js: {
files: ['<%= yeoman.app %>/scripts/{,*/}*.js'],
tasks: ['newer:jshint:all'],
options: {
livereload: '<%= connect.options.livereload %>'
}
},
jsTest: {
files: ['test/spec/{,*/}*.js'],
tasks: ['newer:jshint:test', 'karma']
},
styles: {
files: ['<%= yeoman.app %>/styles/{,*/}*.css'],
tasks: ['newer:copy:styles', 'autoprefixer']
},
gruntfile: {
files: ['Gruntfile.js']
},
livereload: {
options: {
livereload: '<%= connect.options.livereload %>'
},
files: [
'<%= yeoman.app %>/{,*/}*.html',
'.tmp/styles/{,*/}*.css',
'<%= yeoman.app %>/images/{,*/}*.{png,jpg,jpeg,gif,webp,svg}'
]
}
},
// The actual grunt server settings
connect: {
options: {
port: 9000,
// Change this to '0.0.0.0' to access the server from outside.
hostname: 'localhost',
livereload: 35729
},
livereload: {
options: {
open: true,
middleware: function (connect) {
return [
connect.static('.tmp'),
connect().use(
'/bower_components',
connect.static('./bower_components')
),
connect().use(
'/app/styles',
connect.static('./app/styles')
),
connect.static(appConfig.app)
];
}
}
},
test: {
options: {
port: 9001,
middleware: function (connect) {
return [
connect.static('.tmp'),
connect.static('test'),
connect().use(
'/bower_components',
connect.static('./bower_components')
),
connect.static(appConfig.app)
];
}
}
},
dist: {
options: {
open: true,
base: '<%= yeoman.dist %>'
}
}
},
// Make sure code styles are up to par and there are no obvious mistakes
jshint: {
options: {
jshintrc: '.jshintrc',
reporter: require('jshint-stylish')
},
all: {
src: [
'Gruntfile.js',
'<%= yeoman.app %>/scripts/{,*/}*.js'
]
},
test: {
options: {
jshintrc: 'test/.jshintrc'
},
src: ['test/spec/{,*/}*.js']
}
},
// Empties folders to start fresh
clean: {
dist: {
files: [{
dot: true,
src: [
'.tmp',
'<%= yeoman.dist %>/{,*/}*',
'!<%= yeoman.dist %>/.git{,*/}*'
]
}]
},
server: '.tmp'
},
// Add vendor prefixed styles
autoprefixer: {
options: {
browsers: ['last 1 version']
},
server: {
options: {
map: true,
},
files: [{
expand: true,
cwd: '.tmp/styles/',
src: '{,*/}*.css',
dest: '.tmp/styles/'
}]
},
dist: {
files: [{
expand: true,
cwd: '.tmp/styles/',
src: '{,*/}*.css',
dest: '.tmp/styles/'
}]
}
},
// Automatically inject Bower components into the app
wiredep: {
app: {
src: ['<%= yeoman.app %>/index.html'],
ignorePath: /\.\.\//
},
test: {
devDependencies: true,
src: '<%= karma.unit.configFile %>',
ignorePath: /\.\.\//,
fileTypes:{
js: {
block: /(([\s\t]*)\/{2}\s*?bower:\s*?(\S*))(\n|\r|.)*?(\/{2}\s*endbower)/gi,
detect: {
js: /'(.*\.js)'/gi
},
replace: {
js: '\'{{filePath}}\','
}
}
}
}
},
// Renames files for browser caching purposes
filerev: {
dist: {
src: [
'<%= yeoman.dist %>/scripts/{,*/}*.js',
'<%= yeoman.dist %>/styles/{,*/}*.css',
'<%= yeoman.dist %>/images/{,*/}*.{png,jpg,jpeg,gif,webp,svg}',
'<%= yeoman.dist %>/styles/fonts/*'
]
}
},
// Reads HTML for usemin blocks to enable smart builds that automatically
// concat, minify and revision files. Creates configurations in memory so
// additional tasks can operate on them
useminPrepare: {
html: '<%= yeoman.app %>/index.html',
options: {
dest: '<%= yeoman.dist %>',
flow: {
html: {
steps: {
js: ['concat', 'uglifyjs'],
css: ['cssmin']
},
post: {}
}
}
}
},
// Performs rewrites based on filerev and the useminPrepare configuration
usemin: {
html: ['<%= yeoman.dist %>/{,*/}*.html'],
css: ['<%= yeoman.dist %>/styles/{,*/}*.css'],
options: {
assetsDirs: [
'<%= yeoman.dist %>',
'<%= yeoman.dist %>/images',
'<%= yeoman.dist %>/styles'
]
}
},
// The following *-min tasks will produce minified files in the dist folder
// By default, your `index.html`'s <!-- Usemin block --> will take care of
// minification. These next options are pre-configured if you do not wish
// to use the Usemin blocks.
// cssmin: {
// dist: {
// files: {
// '<%= yeoman.dist %>/styles/main.css': [
// '.tmp/styles/{,*/}*.css'
// ]
// }
// }
// },
// uglify: {
// dist: {
// files: {
// '<%= yeoman.dist %>/scripts/scripts.js': [
// '<%= yeoman.dist %>/scripts/scripts.js'
// ]
// }
// }
// },
// concat: {
// dist: {}
// },
imagemin: {
dist: {
files: [{
expand: true,
cwd: '<%= yeoman.app %>/images',
src: '{,*/}*.{png,jpg,jpeg,gif}',
dest: '<%= yeoman.dist %>/images'
}]
}
},
svgmin: {
dist: {
files: [{
expand: true,
cwd: '<%= yeoman.app %>/images',
src: '{,*/}*.svg',
dest: '<%= yeoman.dist %>/images'
}]
}
},
htmlmin: {
dist: {
options: {
collapseWhitespace: true,
conservativeCollapse: true,
collapseBooleanAttributes: true,
removeCommentsFromCDATA: true,
removeOptionalTags: true
},
files: [{
expand: true,
cwd: '<%= yeoman.dist %>',
src: ['*.html', 'views/{,*/}*.html'],
dest: '<%= yeoman.dist %>'
}]
}
},
// ng-annotate tries to make the code safe for minification automatically
// by using the Angular long form for dependency injection.
ngAnnotate: {
dist: {
files: [{
expand: true,
cwd: '.tmp/concat/scripts',
src: '*.js',
dest: '.tmp/concat/scripts'
}]
}
},
// Replace Google CDN references
cdnify: {
dist: {
html: ['<%= yeoman.dist %>/*.html']
}
},
// Copies remaining files to places other tasks can use
copy: {
dist: {
files: [{
expand: true,
dot: true,
cwd: '<%= yeoman.app %>',
dest: '<%= yeoman.dist %>',
src: [
'*.{ico,png,txt}',
'.htaccess',
'*.html',
'views/{,*/}*.html',
'images/{,*/}*.{webp}',
'styles/fonts/{,*/}*.*'
]
}, {
expand: true,
cwd: '.tmp/images',
dest: '<%= yeoman.dist %>/images',
src: ['generated/*']
}]
},
styles: {
expand: true,
cwd: '<%= yeoman.app %>/styles',
dest: '.tmp/styles/',
src: '{,*/}*.css'
}
},
// Run some tasks in parallel to speed up the build process
concurrent: {
server: [
'copy:styles'
],
test: [
'copy:styles'
],
dist: [
'copy:styles',
'imagemin',
'svgmin'
]
},
// Test settings
karma: {
unit: {
configFile: 'test/karma.conf.js',
singleRun: true
}
}
});
grunt.registerTask('serve', 'Compile then start a connect web server', function (target) {
if (target === 'dist') {
return grunt.task.run(['build', 'connect:dist:keepalive']);
}
grunt.task.run([
'clean:server',
'wiredep',
'concurrent:server',
'autoprefixer:server',
'connect:livereload',
'watch'
]);
});
grunt.registerTask('server', 'DEPRECATED TASK. Use the "serve" task instead', function (target) {
grunt.log.warn('The `server` task has been deprecated. Use `grunt serve` to start a server.');
grunt.task.run(['serve:' + target]);
});
grunt.registerTask('test', [
'clean:server',
'wiredep',
'concurrent:test',
'autoprefixer',
'connect:test',
'karma'
]);
grunt.registerTask('build', [
'clean:dist',
'wiredep',
'useminPrepare',
'concurrent:dist',
'autoprefixer',
'concat',
'ngAnnotate',
'copy:dist',
'cdnify',
'cssmin',
'uglify',
'filerev',
'usemin',
'htmlmin'
]);
grunt.registerTask('default', [
'newer:jshint',
'test',
'build'
]
);
};
答案 0 :(得分：1)
您可以编写一个用作管道的自定义Stream实现。如果您随后将GetItemsFromTable()方法调用移动到后台任务中，则客户端可以立即开始从流中读取结果。
在我的解决方案中，我使用循环缓冲区作为管道流的后备存储。仅当客户端足够快地消耗数据时，才会减少内存使用量。但即使在最糟糕的情况下，它也不应该使用比当前解决方案更多的内存。如果内存使用对您而言比执行速度更重要，那么您的流可能会阻止写入调用，直到空间可用。我的解决方案是不块写入;它扩展了循环缓冲区的容量，以便后台线程可以继续填充数据而不会延迟。
GetResults方法可能如下所示：
public Stream GetResults()
{
// Begin filling the pipe with data on a background thread
var pipeStream = new CircularBufferPipeStream();
Task.Run(() => WriteResults(pipeStream));
// Return pipe stream for immediate usage by client
// Note: client is responsible for disposing of the stream after reading all data!
return pipeStream;
}
// Runs on background thread, filling circular buffer with data
void WriteResults(CircularBufferPipeStream stream)
{
IFormatter formatter = new BinaryFormatter();
formatter.Serialize(stream, GetItemsFromTable1());
formatter.Serialize(stream, GetItemsFromTable2());
formatter.Serialize(stream, GetItemsFromTable3());
formatter.Serialize(stream, GetItemsFromTable4());
// Indicate that there's no more data to write
stream.CloseWritePort();
}
循环缓冲流：
/// <summary>
/// Stream that acts as a pipe by supporting reading and writing simultaneously from different threads.
/// Read calls will block until data is available or the CloseWritePort() method has been called.
/// Read calls consume bytes in the circular buffer immediately so that more space is available for writes into the circular buffer.
/// Writes do not block; the capacity of the circular buffer will be expanded as needed to write the entire block of data at once.
/// </summary>
class CircularBufferPipeStream : Stream
{
const int DefaultCapacity = 1024;
byte[] _buffer;
bool _writePortClosed = false;
object _readWriteSyncRoot = new object();
int _length;
ManualResetEvent _dataAddedEvent;
int _start = 0;
public CircularBufferPipeStream(int initialCapacity = DefaultCapacity)
{
_buffer = new byte[initialCapacity];
_length = 0;
_dataAddedEvent = new ManualResetEvent(false);
}
public void CloseWritePort()
{
lock (_readWriteSyncRoot)
{
_writePortClosed = true;
_dataAddedEvent.Set();
}
}
public override bool CanRead { get { return true; } }
public override bool CanWrite { get { return true; } }
public override bool CanSeek { get { return false; } }
public override void Flush() { }
public override long Length { get { throw new NotImplementedException(); } }
public override long Position
{
get { throw new NotImplementedException(); }
set { throw new NotImplementedException(); }
}
public override long Seek(long offset, SeekOrigin origin) { throw new NotImplementedException(); }
public override void SetLength(long value) { throw new NotImplementedException(); }
public override int Read(byte[] buffer, int offset, int count)
{
int bytesRead = 0;
while (bytesRead == 0)
{
bool waitForData = false;
lock (_readWriteSyncRoot)
{
if (_length != 0)
bytesRead = ReadDirect(buffer, offset, count);
else if (_writePortClosed)
break;
else
{
_dataAddedEvent.Reset();
waitForData = true;
}
}
if (waitForData)
_dataAddedEvent.WaitOne();
}
return bytesRead;
}
private int ReadDirect(byte[] buffer, int offset, int count)
{
int readTailCount = Math.Min(Math.Min(_buffer.Length - _start, count), _length);
Array.Copy(_buffer, _start, buffer, offset, readTailCount);
_start += readTailCount;
_length -= readTailCount;
if (_start == _buffer.Length)
_start = 0;
int readHeadCount = Math.Min(Math.Min(_buffer.Length - _start, count - readTailCount), _length);
if (readHeadCount > 0)
{
Array.Copy(_buffer, _start, buffer, offset + readTailCount, readHeadCount);
_start += readHeadCount;
_length -= readHeadCount;
}
return readTailCount + readHeadCount;
}
public override void Write(byte[] buffer, int offset, int count)
{
lock (_readWriteSyncRoot)
{
// expand capacity as needed
if (count + _length > _buffer.Length)
{
var expandedBuffer = new byte[Math.Max(_buffer.Length * 2, count + _length)];
_length = ReadDirect(expandedBuffer, 0, _length);
_start = 0;
_buffer = expandedBuffer;
}
int startWrite = (_start + _length) % _buffer.Length;
int writeTailCount = Math.Min(_buffer.Length - startWrite, count);
Array.Copy(buffer, offset, _buffer, startWrite, writeTailCount);
startWrite += writeTailCount;
_length += writeTailCount;
if (startWrite == _buffer.Length)
startWrite = 0;
int writeHeadCount = count - writeTailCount;
if (writeHeadCount > 0)
{
Array.Copy(buffer, offset + writeTailCount, _buffer, startWrite, writeHeadCount);
_length += writeHeadCount;
}
}
_dataAddedEvent.Set();
}
protected override void Dispose(bool disposing)
{
if (disposing)
{
if (_dataAddedEvent != null)
{
_dataAddedEvent.Dispose();
_dataAddedEvent = null;
}
}
base.Dispose(disposing);
}
}
答案 1 :(得分：0)
尝试
public Stream GetResults()
{
IFormatter formatter = new BinaryFormatter();
Stream stream = new MemoryStream();
formatter.Serialize(stream, GetItemsFromTable1());
formatter.Serialize(stream, GetItemsFromTable2());
formatter.Serialize(stream, GetItemsFromTable3());
formatter.Serialize(stream, GetItemsFromTable4());
stream.Seek(0L, SeekOrigin.Begin);
return stream;
}
为什么要改变？
但是，我不知道如何使用MemoryStream减少内存使用量。我建议将其链接到DeflateStream或FileStream以减少RAM使用量
希望这会有所帮助