00:00:00 stkenmvr149a.fnal.gov 006318 root I LTO4_49MV filemarks written. Tape VOR958 absolute location in blocks 2741380 Thread tape_thread 00:00:00 stkenmvr149a.fnal.gov 006318 root I LTO4_49MV write_tape timing:pull 0.410170316696 write 17.1300120354 crc_check 16.8359465599 freespace 0.427275180817 block_write 34.8034040928 idle 0.671519756317 Thread tape_thread 00:00:00 stkenmvr149a.fnal.gov 006318 root I LTO4_49MV drive stats after write. Tape VOR958 position 112 block 0 block_size 0 bloc_loc 2741380 tot_blocks 762939000 BOT 0 read_err 0 write_err 0 bytes 4217199104 block_write_tot 34.8034040928 tape_rate 121172029.401 Thread tape_thread 00:00:00 stkenmvr149a.fnal.gov 006318 root I LTO4_49MV remaining bytes info in DB 439350272000 estimated 435133072896 reported from drive 441508864000 position 113 Thread tape_thread 00:00:00 stkenmvr149a.fnal.gov 006318 root I LTO4_49MV new bitfile request {'uid': 0, 'external_label': 'VOR958', 'gid': 0, 'location_cookie': '0000_000000000_0000112', 'complete_crc': 3800980574L, 'sanity_cookie': (65536L, 1549018316L), 'size': 4217198571L} Thread tape_thread 00:00:00 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:00 stkenmvr149a.fnal.gov 006318 root I LTO4_49MV set remaining: VOR958 441508864000 0000_000000000_0000113 Thread tape_thread 00:00:00 last message repeated 1 times 00:00:00 stkensrv0n.fnal.gov 028762 enstore I VOLSRV Updating update volume set last_access = '2009-11-04 00:00:00', eod_cookie = '0000_000000000_0000113', non_del_files = '112', remaining_bytes = 441508864000 where label = 'VOR958'; 00:00:00 stkenmvr149a.fnal.gov 006318 root I LTO4_49MV transfer complete volume=VOR958 location=113 Thread tape_thread 00:00:00 last message repeated 1 times 00:00:00 stkensrv0n.fnal.gov 028762 enstore I VOLSRV Updating update volume set sum_wr_access = '112', modification_time = '2009-11-04 00:00:00' where label = 'VOR958'; 00:00:00 stkenmvr149a.fnal.gov 006318 root I LTO4_49MV Sending done to client: {'vc': {'comment': '', 'storage_group': 'cms', 'blocksize': 131072, 'declared': 1255359329.0, 'library': 'CD-LTO4G1', 'si_time': [0.0, 1250979729.0], 'user_inhibit': ['none', 'none'], 'system_inhibit': ['none', 'none'], 'external_label': 'VOR958', 'wrapper': 'cpio_odc', 'remaining_bytes': 439350272000L, 'sum_mounts': 63, 'sum_rd_access': 0, 'file_family_width': 10, 'last_access': 1257314359.0, 'media_type': 'LTO4', 'status': ('ok', None), 'eod_cookie': '0000_000000000_0000112', 'capacity_bytes': 858993459200L, 'non_del_files': 111, 'sum_wr_err': 0, 'sum_wr_access': 111, 'file_family': 'cms11', 'address': ('131.225.13.59', 7502), 'volume_family': 'cms.cms11.cpio_odc', 'modification_time': 1257314359.0, 'write_protected': 'n', 'sum_rd_err': 0, 'first_access': 1257309371.0}, 'outfilepath': '/pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0084/60D8B0F8-D8C8-DE11-A37A-002618943920.root', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 25, 'delayed_dismount': None, 'curpri': 25}, 'file_size': 4217198571L, 'mover': {'max_buffer': 2621440000L, 'check_written_file': 50, 'max_failures': 5, 'library': 'CD-LTO4G1.library_manager', 'local_mover': 0, 'dismount_delay': 30, 'max_consecutive_failures': 3, 'norestart': 'INQ', 'port': 7849, 'mc_device': '0,5,1,3', 'compression': 0, 'serial_num': '1310154355', 'update_interval_in_bound': 2, 'update_interval': 5, 'syslog_entry': 'sense|st[0-9]', 'status': ('ok', None), 'max_rate': 73400320.0, 'vendor_id': 'IBM', 'driver': 'FTTDriver', 'host': 'stkenmvr149a', 'max_time_in_state': 1200, 'device': 'stkenmvr149a:/dev/rmt/tps0d0n', 'name': 'LTO4_49.mover', 'send_stats': 1, 'product_id': 'ULTRIUM-TD4', 'statistics_path': '/tmp/enstore/enstore/LTO4_49.stat', 'single_filemark': 1, 'mount_delay': 15, 'mover_address': ('131.225.13.177', 7849), 'check_first_written_file': 0, 'callback_addr': ('131.225.13.177', 41201), 'data_ip': 'stkenmvr149a', 'logname': 'LTO4_49MV', 'hostip': '131.225.13.177', 'do_cleaning': 'No', 'media_changer': 'SL8500G1.media_changer', 'min_buffer': 67108864L}, 'ignore_fair_share': None, 'retry': None, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0084/60D8B0F8-D8C8-DE11-A37A-002618943920.root', 'uid': 0, 'uname': 'root', 'mtime': 1257312752, 'rmajor': 0, 'machine': ('Linux', 'cmsstor233.fnal.gov', '2.6.31', '#1 SMP Mon Sep 14 09:43:20 CDT 2009', 'x86_64'), 'sanity_size': 65536, 'type': 'cpio_odc', 'pstat': (16893, 121659864, 21L, 1, 9811, 5063, 512, 1257299007, 1257299007, 1257292342), 'mode_octal': '0100036', 'mode': 33188, 'size_bytes': 4217198571L, 'gname': 'root', 'gid': 0, 'fullname': '/storage/data2/write-pool-2/data/000700000000000062450418', 'inode': 121963544L, 'minor': 21}, 'version': 'v3_7d CVS $Revision: 1.922 $ encp', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/storage/data2/write-pool-2/data/000700000000000062450418', 'status': ('ok', None), 'outfile': '/pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0084/.(access)(000700000000000062450418)', 'fc': {'uid': 0, 'external_label': 'VOR958', 'gid': 0, 'location_cookie': '0000_000000000_0000112', 'complete_crc': 3800980574L, 'bfid': 'CDMS125731440000000', 'sanity_cookie': (65536L, 1549018316L), 'size': 4217198571L}, 'infilepath': '/storage/data2/write-pool-2/data/000700000000000062450418', 'at_the_top': 24, 'local_inode': 1467, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.191.105', 46500), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'encp_start_time': 1257312751.221477, 'seek_time': 0.03372502326965332, 'transfer_time': 40.255774021148682, 't0': 1257312751, 'mount_time': 0.038728952407836914, 'lm_dequeued': 1257314360.2529161, 'job_queued': 1257312752.1889451, 'drive_call_time': 16.673543453216553, 'in_queue': 1608.0636298656464, 'drive_transfer_time': 37.4651198387146}, 'user_level': 'admin', 'unique_id': 'cmsst(trunc. 4115) Thread tape_thread 00:00:00 stkenmvr149a.fnal.gov 006318 root I LTO4_49MV Done is sent Thread tape_thread 00:00:00 stkenmvr149a.fnal.gov 006318 root I LTO4_49MV maybe_clean Thread tape_thread 00:00:00 cmsstor233.fnal.gov 031968 root I ENCP Received final dialog for cmsstor233.fnal.gov-1257312752-31968-0. elapsed=1649.430sec 00:00:00 stkenmvr149a.fnal.gov 006318 root I LTO4_49MV returned from maybe_clean Thread tape_thread 00:00:00 cmsstor233.fnal.gov 031968 root I ENCP leaving transfer_file(). 00:00:00 cmsstor233.fnal.gov 031968 root I ENCP pf_status = ('ok', None) skip_layer_cleanup = False 00:00:01 stkensrv0n.fnal.gov 028796 enstore I FILSRV Updating update file set deleted = 'n', pnfs_path = '/pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0084/60D8B0F8-D8C8-DE11-A37A-002618943920.root', drive = 'stkenmvr149a:/dev/rmt/tps0d0n:1310154355', pnfs_id = '000700000000000062450418' where bfid = 'CDMS125731440000000'; 00:00:01 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:01 stkensrv4n.fnal.gov 003689 enstore I SL8500G1MC PROCESSING listDrives 00:00:01 stkensrv4n.fnal.gov 005843 enstore I SL8500F1MC PROCESSING listDrives 00:00:01 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM write_to_hsm next work on vol=VOR958 mover=LTO4_49.mover requester:root 00:00:01 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM HAVE_BOUND:sending write_to_hsm /pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0088/90EF8EB4-F2C8-DE11-80CF-001731AF65EF.root to mover LTO4_49.mover ('131.225.13.177', 7849) DEL_DISM None 00:00:01 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM MSG_TYPE=ADD_TO_LMQ request added to work at movers queue for storage group : cms 00:00:01 stkensrv4n.fnal.gov 005852 enstore I STKMC PROCESSING listDrives 00:00:01 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM HAVE_BOUND:sending {'vc': {'storage_group': 'cms', 'external_label': 'VOR958', 'library': 'CD-LTO4G1', 'file_family': 'cms11', 'wrapper': 'cpio_odc', 'address': ('131.225.13.59', 7502), 'volume_family': 'cms.cms11.cpio_odc', 'file_family_width': 10}, 'outfilepath': '/pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0088/90EF8EB4-F2C8-DE11-80CF-001731AF65EF.root', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 25, 'delayed_dismount': None, 'curpri': 25}, 'file_size': 2816668470L, 'mover': 'LTO4_49.mover', 'ignore_fair_share': None, 'retry': None, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0088/90EF8EB4-F2C8-DE11-80CF-001731AF65EF.root', 'uid': 0, 'uname': 'root', 'type': 'cpio_odc', 'gid': 0, 'mtime': 1257312791, 'rmajor': 0, 'machine': ('Linux', 'cmsstor133.fnal.gov', '2.6.31', '#1 SMP Mon Sep 14 09:43:20 CDT 2009', 'x86_64'), 'sanity_size': 65536, 'pstat': (16893, 123093320, 21L, 1, 9811, 5063, 512, 1257312531, 1257312531, 1257305807), 'mode_octal': '0100036', 'mode': 33188, 'gname': 'root', 'size_bytes': 2816668470L, 'fullname': '/storage/data2/write-pool-2/data/00070000000000006257AFA0', 'inode': 123187104L, 'minor': 21}, 'version': 'v3_7d CVS $Revision: 1.922 $ encp', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/storage/data2/write-pool-2/data/00070000000000006257AFA0', 'status': ('ok', None), 'outfile': '/pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0088/.(access)(00070000000000006257AFA0)', 'fc': {'size': 2816668470L, 'external_label': 'VOR958', 'address': ('131.225.13.59', 7501)}, 'infilepath': '/storage/data2/write-pool-2/data/00070000000000006257AFA0', 'at_the_top': 20, 'local_inode': 851, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.189.104', 58099), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'lm_dequeued': 1257314401.294903, 'encp_start_time': 1257312791.0142181, 'job_queued': 1257312791.9767179, 't0': 1257312791, 'in_queue': 1609.3175859451294}, 'user_level': 'admin', 'unique_id': 'cmsstor133.fnal.gov-1257312791-13052-0'} to mover 00:00:01 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:01 stkenmvr149a.fnal.gov 006318 root I LTO4_49MV WRITE_TO_HSM Thread MainThread 00:00:01 stkenmvr149a.fnal.gov 006318 root I LTO4_49MV SENDING {'vc': {'storage_group': 'cms', 'external_label': 'VOR958', 'library': 'CD-LTO4G1', 'file_family': 'cms11', 'wrapper': 'cpio_odc', 'address': ('131.225.13.59', 7502), 'volume_family': 'cms.cms11.cpio_odc', 'file_family_width': 10}, 'outfilepath': '/pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0088/90EF8EB4-F2C8-DE11-80CF-001731AF65EF.root', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 25, 'delayed_dismount': None, 'curpri': 25}, 'file_size': 2816668470L, 'mover': {'max_buffer': 2621440000L, 'check_written_file': 50, 'max_failures': 5, 'library': 'CD-LTO4G1.library_manager', 'local_mover': 0, 'dismount_delay': 30, 'max_consecutive_failures': 3, 'norestart': 'INQ', 'port': 7849, 'mc_device': '0,5,1,3', 'compression': 0, 'serial_num': '1310154355', 'update_interval_in_bound': 2, 'update_interval': 5, 'syslog_entry': 'sense|st[0-9]', 'status': ('ok', None), 'max_rate': 73400320.0, 'vendor_id': 'IBM', 'driver': 'FTTDriver', 'host': 'stkenmvr149a', 'max_time_in_state': 1200, 'device': 'stkenmvr149a:/dev/rmt/tps0d0n', 'name': 'LTO4_49.mover', 'send_stats': 1, 'product_id': 'ULTRIUM-TD4', 'statistics_path': '/tmp/enstore/enstore/LTO4_49.stat', 'single_filemark': 1, 'mount_delay': 15, 'mover_address': ('131.225.13.177', 7849), 'check_first_written_file': 0, 'callback_addr': ('131.225.13.177', 41204), 'data_ip': 'stkenmvr149a', 'logname': 'LTO4_49MV', 'hostip': '131.225.13.177', 'do_cleaning': 'No', 'media_changer': 'SL8500G1.media_changer', 'min_buffer': 67108864L}, 'ignore_fair_share': None, 'retry': None, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0088/90EF8EB4-F2C8-DE11-80CF-001731AF65EF.root', 'uid': 0, 'uname': 'root', 'mtime': 1257312791, 'rmajor': 0, 'machine': ('Linux', 'cmsstor133.fnal.gov', '2.6.31', '#1 SMP Mon Sep 14 09:43:20 CDT 2009', 'x86_64'), 'sanity_size': 65536, 'type': 'cpio_odc', 'pstat': (16893, 123093320, 21L, 1, 9811, 5063, 512, 1257312531, 1257312531, 1257305807), 'mode_octal': '0100036', 'mode': 33188, 'size_bytes': 2816668470L, 'gname': 'root', 'gid': 0, 'fullname': '/storage/data2/write-pool-2/data/00070000000000006257AFA0', 'inode': 123187104L, 'minor': 21}, 'version': 'v3_7d CVS $Revision: 1.922 $ encp', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/storage/data2/write-pool-2/data/00070000000000006257AFA0', 'status': ('ok', None), 'outfile': '/pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0088/.(access)(00070000000000006257AFA0)', 'fc': {'size': 2816668470L, 'external_label': 'VOR958', 'address': ('131.225.13.59', 7501)}, 'infilepath': '/storage/data2/write-pool-2/data/00070000000000006257AFA0', 'at_the_top': 20, 'local_inode': 851, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.189.104', 58099), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'lm_dequeued': 1257314401.294903, 'encp_start_time': 1257312791.0142181, 'job_queued': 1257312791.9767179, 't0': 1257312791, 'in_queue': 1609.3175859451294}, 'user_level': 'admin', 'unique_id': 'cmsstor133.fnal.gov-1257312791-13052-0'} Thread client_connect_thread 00:00:01 cmsstor133.fnal.gov 013052 root I ENCP Control socket ('131.225.189.104', 58099) is connected to ('131.225.13.177', 41205) for cmsstor133.fnal.gov-1257312791-13052-0. elapsed=1610.170sec 00:00:01 cmsstor233.fnal.gov 031968 root I ENCP MSG_TYPE=ENCP_XFER write_to_hsm /storage/data2/write-pool-2/data/000700000000000062450418 -> /pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0084/60D8B0F8-D8C8-DE11-A37A-002618943920.root: 4217198571 bytes copied to VOR958 at 99.9 MB/S (94.6 MB/S network) (107 MB/S drive) (112 MB/S disk) mover=LTO4_49.mover drive_id=ULTRIUM-TD4 drive_sn=1310154355 drive_vendor=IBM elapsed=1650.1 {'storage_group': 'cms', 'encp_crc': 3800980574L, 'disk_rate': 111.56565482744801, 'drive_rate': 107.34875483280518, 'driver': 'FTTDriver', 'transfer_rate': 99.88679644943673, 'mover_interface': 'stkenmvr149a', 'overall_rate': 98.0150156256515, 'media_changer': 'SL8500G1.media_changer', 'network_rate': 99.88679644943673, 'encp_ip': '131.225.191.105', 'unique_id': 'cmsstor233.fnal.gov-1257312752-31968-0'} 00:00:01 cmsstor233.fnal.gov 031968 root I ENCP File /pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0084/60D8B0F8-D8C8-DE11-A37A-002618943920.root copied successfully. 00:00:01 cmsstor133.fnal.gov 013052 root I ENCP Received callback ticket from mover LTO4_49.mover for transfer cmsstor133.fnal.gov-1257312791-13052-0. elapsed=1610.170sec 00:00:01 cmsstor133.fnal.gov 013052 root I ENCP Atempting to connect data socket to mover at ('131.225.13.177', 41204). 00:00:01 cmsstor133.fnal.gov 013052 root I ENCP Connecting data socket to mover (('131.225.13.177', 41204)) with interface 131.225.189.104. elapsed=1610.180sec 00:00:01 cmsstor233.fnal.gov 031968 root I ENCP Completed transferring 4217198571 bytes in 1 files in 1650.10948992 sec. Overall rate = 98 MB/sec. Transfer rate = 99.9 MB/sec. Network rate = 94.6 MB/sec. Drive rate = 107 MB/sec. Disk rate = 112 MB/sec. Exit status = 0. 00:00:01 cmsstor133.fnal.gov 013052 root I ENCP Data socket ('131.225.189.104', 46181) is connected to ('131.225.13.177', 41204). elapsed=1610.180sec 00:00:01 cmsstor133.fnal.gov 013052 root I ENCP Opening local file. 00:00:01 cmsstor133.fnal.gov 013052 root I ENCP pf_status = ('ok', None) skip_layer_cleanup = False 00:00:01 stkenmvr149a.fnal.gov 006318 root I LTO4_49MV tape position: target 113 current 113 0 Thread seek_thread 00:00:01 stkenmvr149a.fnal.gov 006318 root I LTO4_49MV write_tape starting, bytes_to_write=2816669184 Thread tape_thread 00:00:01 stkenmvr149a.fnal.gov 006318 root I LTO4_49MV Write starting. Tape VOR958 absolute location in blocks 2741380 Thread tape_thread 00:00:02 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM 00:00:02 cmsstor152.fnal.gov 002606 root I ENCP File /storage/data1/write-pool-1/data/000700000000000062429FA8 transfered for cmsstor152.fnal.gov-1257312752-2606-0 in 32.434623003 seconds. 00:00:02 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:02 cmsstor70.fnal.gov 003542 root I ENCP Version: v3_7d CVS $Revision: 1.922 $ encp OS: Linux 2.6.31 x86_64 Release: Scientific Linux SL release 5.2 (Boron) Rocks release 5.1 (V.I) User: root(0) Group: root(0) Euser: root(0) Egroup: root(0) Library: CD-LTO4G1 Storage Group: cms File Family: cms11 FF Wrapper: cpio_odc FF Width: 10 Current working directory: cmsstor70.fnal.gov:/ Command line: /home/enstore/enstore/bin/encp --verbose 4 --threaded --ecrc --resubmit-timeout 1800 --priority 25 --pnfs-mount /pnfs/fs --put-cache 00070000000000006253A898 /storage/data2/write-pool-2/data/00070000000000006253A898 00:00:02 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:02 stkenmvr195a.fnal.gov 021479 root I LTO4_95MV written bytes 5832559616/5832559616, blocks 44499 header 192 trailer 194 Thread tape_thread 00:00:03 cdfenmvr192a.fnal.gov 006151 root I LTO4_92MV filemarks written. Tape VOV138 absolute location in blocks 255370 Thread tape_thread 00:00:03 cdfenmvr192a.fnal.gov 006151 root I LTO4_92MV write_tape timing:pull 4.17232513428e-05 write 0.00510287284851 crc_check 0.00296926498413 freespace 3.69548797607e-05 block_write 0.00815081596375 idle 8.20159912109e-05 Thread tape_thread 00:00:03 cdfenmvr192a.fnal.gov 006151 root I LTO4_92MV drive stats after write. Tape VOV138 position 5173 block 0 block_size 0 bloc_loc 255370 tot_blocks 762939000 BOT 0 read_err 0 write_err 14 bytes 589824 block_write_tot 0.00815081596375 tape_rate 72363798.0079 Thread tape_thread 00:00:03 cdfenmvr192a.fnal.gov 006151 root I LTO4_92MV remaining bytes info in DB 738947072000 estimated 738946482176 reported from drive 738942976000 position 5176 Thread tape_thread 00:00:03 cdfenmvr192a.fnal.gov 006151 root I LTO4_92MV new bitfile request {'uid': 0, 'external_label': 'VOV138', 'gid': 0, 'location_cookie': '0000_000000000_0005174', 'complete_crc': 1037339544L, 'sanity_cookie': (65536L, 4191396787L), 'size': 589768L} Thread tape_thread 00:00:03 cmsstor70.fnal.gov 003542 root I ENCP Creating zero length output files. 00:00:03 cdfenmvr192a.fnal.gov 006151 root I LTO4_92MV set remaining: VOV138 738942976000 0000_000000000_0005176 Thread tape_thread 00:00:03 last message repeated 1 times 00:00:03 stkensrv0n.fnal.gov 028762 enstore I VOLSRV Updating update volume set last_access = '2009-11-04 00:00:03', eod_cookie = '0000_000000000_0005176', non_del_files = '1726', remaining_bytes = 738942976000 where label = 'VOV138'; 00:00:03 cmsstor70.fnal.gov 003542 root I ENCP Sending /pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0086/0882E6EC-EEC8-DE11-B0BB-002618943944.root write request to LM: unique_id: cmsstor70.fnal.gov-1257314402-3542-0 inputfile: /storage/data2/write-pool-2/data/00070000000000006253A898 outputfile: /pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0086/.(access)(00070000000000006253A898) 00:00:03 cdfenmvr192a.fnal.gov 006151 root I LTO4_92MV transfer complete volume=VOV138 location=5176 Thread tape_thread 00:00:03 last message repeated 1 times 00:00:03 stkensrv0n.fnal.gov 028762 enstore I VOLSRV Updating update volume set sum_wr_access = '1725', modification_time = '2009-11-04 00:00:03' where label = 'VOV138'; 00:00:03 cdfenmvr192a.fnal.gov 006151 root I LTO4_92MV Sending done to client: {'vc': {'comment': '', 'storage_group': 'ktev', 'blocksize': 131072, 'declared': 1254165875.0, 'library': 'CD-LTO4F1', 'si_time': [0.0, 0.0], 'user_inhibit': ['none', 'none'], 'system_inhibit': ['none', 'none'], 'external_label': 'VOV138', 'wrapper': 'cern', 'remaining_bytes': 738947072000L, 'sum_mounts': 1, 'sum_rd_access': 0, 'file_family_width': 1, 'last_access': 1257314397.0, 'media_type': 'LTO4', 'status': ('ok', None), 'eod_cookie': '0000_000000000_0005173', 'capacity_bytes': 858993459200L, 'non_del_files': 1725, 'sum_wr_err': 0, 'sum_wr_access': 1724, 'file_family': 'ktev_migration-MIGRATION', 'address': ('131.225.13.59', 7502), 'volume_family': 'ktev.ktev_migration-MIGRATION.cern', 'modification_time': 1257314397.0, 'write_protected': 'n', 'sum_rd_err': 0, 'first_access': 1257302588.0}, 'outfilepath': '/pnfs/fs/usr/Migration/ktev/migrated_from_dlt/KQ/KQRC/KQRC52/f1671', 'encp': {'agetime': 30, 'adminpri': 6, 'delpri': 1, 'basepri': 1000007, 'delayed_dismount': 2, 'curpri': 1000007}, 'file_size': 589768L, 'mover': {'max_buffer': 2621440000L, 'check_written_file': 600, 'max_failures': 5, 'library': 'CD-LTO4F1.library_manager', 'local_mover': 0, 'single_filemark': 1, 'max_consecutive_failures': 3, 'norestart': 'INQ', 'port': 7853, 'mc_device': '1,0,1,1', 'compression': 0, 'serial_num': '1310060800', 'update_interval_in_bound': 2, 'update_interval': 5, 'syslog_entry': 'sense|st[0-9]', 'status': ('ok', None), 'max_rate': 73400320.0, 'vendor_id': 'IBM', 'driver': 'FTTDriver', 'host': 'cdfenmvr192a', 'max_time_in_state': 1200, 'device': 'cdfenmvr192a:/dev/rmt/tps0d0n', 'name': 'LTO4_92.mover', 'send_stats': 1, 'product_id': 'ULTRIUM-TD4', 'statistics_path': '/tmp/enstore/enstore/LTO4_92.stat', 'mover_address': ('131.225.215.76', 7853), 'mount_delay': 15, 'check_first_written_file': 0, 'callback_addr': ('131.225.215.76', 60645), 'data_ip': 'cdfenmvr192a', 'logname': 'LTO4_92MV', 'hostip': '131.225.215.76', 'do_cleaning': 'No', 'media_changer': 'SL8500F1.media_changer'}, 'ignore_fair_share': 1, 'retry': None, 'lm': {'address': ('131.225.13.4', 8542)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fs/usr/ktev/migrated_from_dlt/KQ/KQRC/KQRC52/f1671', 'uid': 0, 'uname': 'root', 'mtime': 1257314385, 'rmajor': 0, 'machine': ('Linux', 'stkendm7a.fnal.gov', '2.6.9-89.0.9.ELsmp', '#1 SMP Mon Aug 24 13:47:43 CDT 2009', 'i686'), 'sanity_size': 65536, 'type': 'cern', 'pstat': (16877, 678574248L, 20L, 1, 0, 0, 512L, 1257314311, 1257314311, 1252942658), 'mode_octal': '0100016', 'mode': 33188, 'size_bytes': 589768L, 'gname': 'root', 'gid': 0, 'fullname': '/data/data2/Migration_Spool/VO6153:0000_000000000_0032291', 'inode': 675389672L, 'minor': 20}, 'version': 'v3_8 CVS $Revision: 1.963 $ encp.pyc', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/data/data2/Migration_Spool/VO6153:0000_000000000_0032291', 'status': ('ok', None), 'outfile': '/pnfs/fs/usr/Migration/ktev/migrated_from_dlt/KQ/KQRC/KQRC52/.(access)(00280000000000000841A0E8)', 'fc': {'uid': 0, 'external_label': 'VOV138', 'gid': 0, 'location_cookie': '0000_000000000_0005174', 'complete_crc': 1037339544L, 'bfid': 'CDMS125731440300000', 'sanity_cookie': (65536L, 4191396787L), 'size': 589768L}, 'infilepath': '/data/data2/Migration_Spool/VO6153:0000_000000000_0032291', 'at_the_top': 36, 'local_inode': 4495344L, 'resend': {'max_retry': 3, 'resubmits': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.13.19', 57433), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'encp_start_time': 1257314385.7322309, 'seek_time': 0.026814937591552734, 'transfer_time': 5.648730993270874, 't0': 1257314385, 'mount_time': 0.015924930572509766, 'lm_dequeued': 1257314397.367893, 'job_queued': 1257314385.935122, 'drive_call_time': 0.016037225723266602, 'in_queue': 11.432508945465088, 'drive_transfer_time': 5.5428733825683594}, 'user_level': 'admin', 'unique_id': 'stkendm7a.fnal.gov-1257314385-18643-24629'} Thread tape_thread 00:00:03 cdfenmvr192a.fnal.gov 006151 root I LTO4_92MV Done is sent Thread tape_thread 00:00:03 cdfenmvr192a.fnal.gov 006151 root I LTO4_92MV maybe_clean Thread tape_thread 00:00:03 stkendm7a.fnal.gov 018643 root I ENCP Received final dialog for stkendm7a.fnal.gov-1257314385-18643-24629. elapsed=154649.540sec Thread WRITE_2 00:00:03 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM MSG_TYPE=ADD_TO_LMQ request added to pending queue for storage group : cms 00:00:03 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM write Q'd /storage/data2/write-pool-2/data/00070000000000006253A898 (cmsstor70.fnal.gov-1257314402-3542-0) -> /pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0086/0882E6EC-EEC8-DE11-B0BB-002618943944.root : library=CD-LTO4G1 family=cms11 requester:root volume_family:cms.cms11.cpio_odc 00:00:03 stkendm7a.fnal.gov 018643 root I ENCP leaving transfer_file(). Thread WRITE_2 00:00:03 cdfenmvr192a.fnal.gov 006151 root I LTO4_92MV returned from maybe_clean Thread tape_thread 00:00:03 cmsstor70.fnal.gov 003542 root I ENCP Listening for control socket at: ('131.225.189.70', 45049) 00:00:03 cmsstor70.fnal.gov 003542 root I ENCP Waiting for mover to connect control socket at address: ('131.225.189.70', 45049) elapsed=2.390sec 00:00:03 stkensrv0n.fnal.gov 028796 enstore I FILSRV Updating update file set deleted = 'n', pnfs_path = '/pnfs/fs/usr/ktev/migrated_from_dlt/KQ/KQRC/KQRC52/f1671', drive = 'cdfenmvr192a:/dev/rmt/tps0d0n:1310060800', pnfs_id = '00280000000000000841A0E8' where bfid = 'CDMS125731440300000'; 00:00:03 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM 00:00:03 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM write_to_hsm next work on vol=VOV138 mover=LTO4_92.mover requester:root 00:00:03 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM HAVE_BOUND:sending write_to_hsm /pnfs/fs/usr/ktev/migrated_from_dlt/KQ/KQH0/KQH054/f336 to mover LTO4_92.mover ('131.225.215.76', 7853) DEL_DISM 2 00:00:03 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM MSG_TYPE=ADD_TO_LMQ request added to work at movers queue for storage group : ktev 00:00:03 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM HAVE_BOUND:sending {'vc': {'storage_group': 'ktev', 'external_label': 'VOV138', 'library': 'CD-LTO4F1', 'file_family': 'ktev_migration-MIGRATION', 'wrapper': 'cern', 'address': ('131.225.13.59', 7502), 'volume_family': 'ktev.ktev_migration-MIGRATION.cern', 'file_family_width': 1}, 'outfilepath': '/pnfs/fs/usr/Migration/ktev/migrated_from_dlt/KQ/KQH0/KQH054/f336', 'encp': {'agetime': 30, 'adminpri': 6, 'delpri': 1, 'basepri': 1000007, 'delayed_dismount': 2, 'curpri': 1000007}, 'file_size': 9447800L, 'mover': 'LTO4_92.mover', 'ignore_fair_share': 1, 'retry': None, 'lm': {'address': ('131.225.13.4', 8542)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fs/usr/ktev/migrated_from_dlt/KQ/KQH0/KQH054/f336', 'uid': 0, 'uname': 'root', 'type': 'cern', 'gid': 0, 'mtime': 1257314391, 'rmajor': 0, 'machine': ('Linux', 'stkendm7a.fnal.gov', '2.6.9-89.0.9.ELsmp', '#1 SMP Mon Aug 24 13:47:43 CDT 2009', 'i686'), 'sanity_size': 65536, 'pstat': (16877, 686516192L, 20L, 1, 0, 0, 512L, 1257313751, 1257313751, 1255446031), 'mode_octal': '0100016', 'mode': 33188, 'gname': 'root', 'size_bytes': 9447800L, 'fullname': '/data/data2/Migration_Spool/VO6153:0000_000000000_0031880', 'inode': 675389696L, 'minor': 20}, 'version': 'v3_8 CVS $Revision: 1.963 $ encp.pyc', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/data/data2/Migration_Spool/VO6153:0000_000000000_0031880', 'status': ('ok', None), 'outfile': '/pnfs/fs/usr/Migration/ktev/migrated_from_dlt/KQ/KQH0/KQH054/.(access)(00280000000000000841A100)', 'fc': {'size': 9447800L, 'external_label': 'VOV138', 'pnfsid': '00280000000000000841A100', 'address': ('131.225.13.59', 7501)}, 'infilepath': '/data/data2/Migration_Spool/VO6153:0000_000000000_0031880', 'at_the_top': 36, 'local_inode': 33308833L, 'resend': {'max_retry': 3, 'resubmits': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.13.19', 57436), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'lm_dequeued': 1257314403.378454, 'encp_start_time': 1257314391.778286, 'job_queued': 1257314392.013905, 't0': 1257314391, 'in_queue': 11.364289045333862}, 'user_level': 'admin', 'unique_id': 'stkendm7a.fnal.gov-1257314391-18643-24630'} to mover 00:00:03 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM 00:00:03 cdfenmvr192a.fnal.gov 006151 root I LTO4_92MV WRITE_TO_HSM Thread MainThread 00:00:03 cdfenmvr192a.fnal.gov 006151 root I LTO4_92MV SENDING {'vc': {'storage_group': 'ktev', 'external_label': 'VOV138', 'library': 'CD-LTO4F1', 'file_family': 'ktev_migration-MIGRATION', 'wrapper': 'cern', 'address': ('131.225.13.59', 7502), 'volume_family': 'ktev.ktev_migration-MIGRATION.cern', 'file_family_width': 1}, 'outfilepath': '/pnfs/fs/usr/Migration/ktev/migrated_from_dlt/KQ/KQH0/KQH054/f336', 'encp': {'agetime': 30, 'adminpri': 6, 'delpri': 1, 'basepri': 1000007, 'delayed_dismount': 2, 'curpri': 1000007}, 'file_size': 9447800L, 'mover': {'max_buffer': 2621440000L, 'check_written_file': 600, 'max_failures': 5, 'library': 'CD-LTO4F1.library_manager', 'local_mover': 0, 'single_filemark': 1, 'max_consecutive_failures': 3, 'norestart': 'INQ', 'port': 7853, 'mc_device': '1,0,1,1', 'compression': 0, 'serial_num': '1310060800', 'update_interval_in_bound': 2, 'update_interval': 5, 'syslog_entry': 'sense|st[0-9]', 'status': ('ok', None), 'max_rate': 73400320.0, 'vendor_id': 'IBM', 'driver': 'FTTDriver', 'host': 'cdfenmvr192a', 'max_time_in_state': 1200, 'device': 'cdfenmvr192a:/dev/rmt/tps0d0n', 'name': 'LTO4_92.mover', 'send_stats': 1, 'product_id': 'ULTRIUM-TD4', 'statistics_path': '/tmp/enstore/enstore/LTO4_92.stat', 'mover_address': ('131.225.215.76', 7853), 'mount_delay': 15, 'check_first_written_file': 0, 'callback_addr': ('131.225.215.76', 60647), 'data_ip': 'cdfenmvr192a', 'logname': 'LTO4_92MV', 'hostip': '131.225.215.76', 'do_cleaning': 'No', 'media_changer': 'SL8500F1.media_changer'}, 'ignore_fair_share': 1, 'retry': None, 'lm': {'address': ('131.225.13.4', 8542)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fs/usr/ktev/migrated_from_dlt/KQ/KQH0/KQH054/f336', 'uid': 0, 'uname': 'root', 'mtime': 1257314391, 'rmajor': 0, 'machine': ('Linux', 'stkendm7a.fnal.gov', '2.6.9-89.0.9.ELsmp', '#1 SMP Mon Aug 24 13:47:43 CDT 2009', 'i686'), 'sanity_size': 65536, 'type': 'cern', 'pstat': (16877, 686516192L, 20L, 1, 0, 0, 512L, 1257313751, 1257313751, 1255446031), 'mode_octal': '0100016', 'mode': 33188, 'size_bytes': 9447800L, 'gname': 'root', 'gid': 0, 'fullname': '/data/data2/Migration_Spool/VO6153:0000_000000000_0031880', 'inode': 675389696L, 'minor': 20}, 'version': 'v3_8 CVS $Revision: 1.963 $ encp.pyc', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/data/data2/Migration_Spool/VO6153:0000_000000000_0031880', 'status': ('ok', None), 'outfile': '/pnfs/fs/usr/Migration/ktev/migrated_from_dlt/KQ/KQH0/KQH054/.(access)(00280000000000000841A100)', 'fc': {'size': 9447800L, 'external_label': 'VOV138', 'pnfsid': '00280000000000000841A100', 'address': ('131.225.13.59', 7501)}, 'infilepath': '/data/data2/Migration_Spool/VO6153:0000_000000000_0031880', 'at_the_top': 36, 'local_inode': 33308833L, 'resend': {'max_retry': 3, 'resubmits': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.13.19', 57436), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'lm_dequeued': 1257314403.378454, 'encp_start_time': 1257314391.778286, 'job_queued': 1257314392.013905, 't0': 1257314391, 'in_queue': 11.364289045333862}, 'user_level': 'admin', 'unique_id': 'stkendm7a.fnal.gov-1257314391-18643-24630'} Thread client_connect_thread 00:00:03 stkendm7a.fnal.gov 018643 root I ENCP Listening for control socket at: ('131.225.13.19', 57436) Thread WRITE 00:00:03 stkendm7a.fnal.gov 018643 root I ENCP Waiting for mover to connect control socket at address: ('131.225.13.19', 57436) elapsed=154649.840sec Thread WRITE 00:00:03 stkendm7a.fnal.gov 018643 root I ENCP Control socket ('131.225.13.19', 57436) is connected to ('131.225.215.76', 60648) for stkendm7a.fnal.gov-1257314391-18643-24630. elapsed=154649.840sec Thread WRITE 00:00:03 stkendm7a.fnal.gov 018643 root I ENCP Received callback ticket from mover LTO4_92.mover for transfer stkendm7a.fnal.gov-1257314391-18643-24630. elapsed=154649.850sec Thread WRITE 00:00:03 stkendm7a.fnal.gov 018643 root I ENCP Attempting to connect data socket to mover at ('131.225.215.76', 60647). Thread WRITE 00:00:03 stkendm7a.fnal.gov 018643 root I ENCP Connecting data socket to mover (('131.225.215.76', 60647)) with interface 131.225.13.19. elapsed=154649.850sec Thread WRITE 00:00:03 stkendm7a.fnal.gov 018643 root I ENCP Data socket ('131.225.13.19', 57440) is connected to ('131.225.215.76', 60647). elapsed=154649.850sec Thread WRITE 00:00:03 stkendm7a.fnal.gov 018643 root I ENCP Opening local file. Thread WRITE 00:00:03 stkendm7a.fnal.gov 018643 root I ENCP pf_status = ('ok', None) skip_layer_cleanup = False Thread WRITE 00:00:03 cdfenmvr192a.fnal.gov 006151 root I LTO4_92MV tape position: target 5176 current 5176 0 Thread seek_thread 00:00:03 cdfenmvr192a.fnal.gov 006151 root I LTO4_92MV write_tape starting, bytes_to_write=9447936 Thread tape_thread 00:00:03 cdfenmvr192a.fnal.gov 006151 root I LTO4_92MV Write starting. Tape VOV138 absolute location in blocks 255370 Thread tape_thread 00:00:03 stkendm7a.fnal.gov 018643 root I ENCP MSG_TYPE=ENCP_XFER write_to_hsm /data/data2/Migration_Spool/VO6153:0000_000000000_0032291 -> /pnfs/fs/usr/Migration/ktev/migrated_from_dlt/KQ/KQRC/KQRC52/f1671: 589768 bytes copied to VOV138 at 0.0995 MB/S (88.2 MB/S network) (0.101 MB/S drive) (21.9 MB/S disk) mover=LTO4_92.mover drive_id=ULTRIUM-TD4 drive_sn=1310060800 drive_vendor=IBM elapsed=17.825 {'storage_group': 'ktev', 'encp_crc': 1037339544L, 'disk_rate': 21.94057313685509, 'drive_rate': 0.10147202640549306, 'driver': 'FTTDriver', 'transfer_rate': 0.099534048043901313, 'mover_interface': 'cdfenmvr192a', 'overall_rate': 0.091410198943440393, 'media_changer': 'SL8500F1.media_changer', 'network_rate': 0.099534048043901313, 'encp_ip': '131.225.13.19', 'unique_id': 'stkendm7a.fnal.gov-1257314385-18643-24629'} Thread WRITE_2 00:00:03 stkendm7a.fnal.gov 018643 root I ENCP File /pnfs/fs/usr/Migration/ktev/migrated_from_dlt/KQ/KQRC/KQRC52/f1671 copied successfully. Thread WRITE_2 00:00:03 stkendm7a.fnal.gov 018643 root I ENCP Completed transferring 589768 bytes in 1 files in 17.8275191784 sec. Overall rate = 0.0914 MB/sec. Transfer rate = 0.0995 MB/sec. Network rate = 88.2 MB/sec. Drive rate = 0.101 MB/sec. Disk rate = 21.9 MB/sec. Exit status = 0. Thread WRITE_2 00:00:03 stkendm7a.fnal.gov 018643 root I ENCP File /data/data2/Migration_Spool/VO6153:0000_000000000_0031880 transfered for stkendm7a.fnal.gov-1257314391-18643-24630 in 0.104156970978 seconds. Thread WRITE 00:00:03 stkensrv4n.fnal.gov 004149 enstore I SL8500G1MC query drive 0,6,1,2 => 0,0,6,1,2 online in use VOT806 IBM-LTO4 00:00:03 stkensrv0n.fnal.gov 028796 enstore I FILSRV start modifying {'pnfsid': '00280000000000000841A0E8', 'pnfs_name0': '/pnfs/fs/usr/ktev/migrated_from_dlt/KQ/KQRC/KQRC52/f1671', 'deleted': 'no', 'external_label': 'VOV138', 'drive': 'cdfenmvr192a:/dev/rmt/tps0d0n:1310060800', 'update': '2009-11-04 00:00:03.226457', 'gid': 0, 'location_cookie': '0000_000000000_0005174', 'complete_crc': 1037339544L, 'uid': 0, 'bfid': 'CDMS125731440300000', 'sanity_cookie': (65536L, 4191396787L), 'size': 589768L} 00:00:03 stkensrv0n.fnal.gov 028796 enstore I FILSRV Updating update file set pnfs_id = '001F000000000000010244C0' where bfid = 'CDMS125731440300000'; 00:00:03 stkensrv0n.fnal.gov 028796 enstore I FILSRV modified to {'pnfsid': '001F000000000000010244C0', 'pnfs_name0': '/pnfs/fs/usr/ktev/migrated_from_dlt/KQ/KQRC/KQRC52/f1671', 'deleted': 'no', 'external_label': 'VOV138', 'drive': 'cdfenmvr192a:/dev/rmt/tps0d0n:1310060800', 'update': '2009-11-04 00:00:03.226457', 'gid': 0, 'location_cookie': '0000_000000000_0005174', 'complete_crc': 1037339544L, 'uid': 0, 'bfid': 'CDMS125731440300000', 'sanity_cookie': (65536L, 4191396787L), 'size': 589768L} 00:00:04 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM 00:00:04 stkendm7a.fnal.gov 018643 root I ENCP Version: v3_8 CVS $Revision: 1.963 $ encp.pyc OS: Linux 2.6.9-89.0.9.ELsmp i686 Release: Scientific Linux Fermi LTS release 4.4 (Wilson) User: root(0) Group: root(0) Euser: root(0) Egroup: root(0) Library: CD-LTO4F1 Storage Group: ktev File Family: ktev_migration-MIGRATION FF Wrapper: cern FF Width: 1 Current working directory: stkendm7a.fnal.gov:/opt/enstore/src Command line: encp --delayed-dismount 2 --ignore-fair-share --threaded --priority 0 --storage-group ktev --file-family ktev_migration-MIGRATION --file-family-wrapper cern --library CD-LTO4F1 --override-path /pnfs/fs/usr/ktev/migrated_from_dlt/KQ/KQRC/KQRC52/f5012 --threaded /data/data2/Migration_Spool/VO6153:0000_000000000_0032300 /pnfs/fs/usr/Migration/ktev/migrated_from_dlt/KQ/KQRC/KQRC52/f5012 Thread WRITE_2 00:00:04 stkenmvr193a.fnal.gov 031647 root I LTO4_93MV written bytes 1966080512/1966080512, blocks 15001 header 188 trailer 324 Thread tape_thread 00:00:04 stkendm7a.fnal.gov 018643 root I ENCP Creating zero length output files. Thread WRITE_2 00:00:04 stkenmvr195a.fnal.gov 021479 root I LTO4_95MV filemarks written. Tape VOR957 absolute location in blocks 3935347 Thread tape_thread 00:00:04 stkenmvr195a.fnal.gov 021479 root I LTO4_95MV write_tape timing:pull 0.820819616318 write 19.4503917694 crc_check 26.4282884598 freespace 0.432559967041 block_write 47.1320598125 idle 0.797794342041 Thread tape_thread 00:00:04 stkenmvr195a.fnal.gov 021479 root I LTO4_95MV drive stats after write. Tape VOR957 position 170 block 0 block_size 0 bloc_loc 3935347 tot_blocks 762939000 BOT 0 read_err 0 write_err 0 bytes 5832559616 block_write_tot 47.1320598125 tape_rate 123749304.384 Thread tape_thread 00:00:04 stkenmvr195a.fnal.gov 021479 root I LTO4_95MV remaining bytes info in DB 306467840000 estimated 300635280384 reported from drive 299365376000 position 171 Thread tape_thread 00:00:04 stkenmvr195a.fnal.gov 021479 root I LTO4_95MV new bitfile request {'uid': 0, 'external_label': 'VOR957', 'gid': 0, 'location_cookie': '0000_000000000_0000170', 'complete_crc': 3986614447L, 'sanity_cookie': (65536L, 1824181559L), 'size': 5832559230L} Thread tape_thread 00:00:04 stkendm7a.fnal.gov 018643 root I ENCP Sending /pnfs/fs/usr/Migration/ktev/migrated_from_dlt/KQ/KQRC/KQRC52/f5012 write request to LM: unique_id: stkendm7a.fnal.gov-1257314404-18643-24632 inputfile: /data/data2/Migration_Spool/VO6153:0000_000000000_0032300 outputfile: /pnfs/fs/usr/Migration/ktev/migrated_from_dlt/KQ/KQRC/KQRC52/.(access)(00280000000000000841A148) Thread WRITE_2 00:00:04 stkenmvr195a.fnal.gov 021479 root I LTO4_95MV set remaining: VOR957 299365376000 0000_000000000_0000171 Thread tape_thread 00:00:04 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM MSG_TYPE=ADD_TO_LMQ request added to pending queue for storage group : ktev 00:00:04 last message repeated 1 times 00:00:04 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM write Q'd /data/data2/Migration_Spool/VO6153:0000_000000000_0032300 (stkendm7a.fnal.gov-1257314404-18643-24632) -> /pnfs/fs/usr/ktev/migrated_from_dlt/KQ/KQRC/KQRC52/f5012 : library=CD-LTO4F1 family=ktev_migration-MIGRATION requester:root volume_family:ktev.ktev_migration-MIGRATION.cern 00:00:04 stkensrv0n.fnal.gov 028762 enstore I VOLSRV Updating update volume set last_access = '2009-11-04 00:00:04', eod_cookie = '0000_000000000_0000171', non_del_files = '170', remaining_bytes = 299365376000 where label = 'VOR957'; 00:00:04 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM 00:00:04 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:04 stkenmvr195a.fnal.gov 021479 root I LTO4_95MV transfer complete volume=VOR957 location=171 Thread tape_thread 00:00:04 last message repeated 1 times 00:00:04 stkensrv0n.fnal.gov 028762 enstore I VOLSRV Updating update volume set sum_wr_access = '170', modification_time = '2009-11-04 00:00:04' where label = 'VOR957'; 00:00:04 stkenmvr195a.fnal.gov 021479 root I LTO4_95MV Sending done to client: {'vc': {'comment': '', 'storage_group': 'cms', 'blocksize': 131072, 'declared': 1255359326.0, 'library': 'CD-LTO4G1', 'si_time': [0.0, 1250983192.0], 'user_inhibit': ['none', 'none'], 'system_inhibit': ['none', 'none'], 'external_label': 'VOR957', 'wrapper': 'cpio_odc', 'remaining_bytes': 306467840000L, 'sum_mounts': 74, 'sum_rd_access': 0, 'file_family_width': 10, 'last_access': 1257314339.0, 'media_type': 'LTO4', 'status': ('ok', None), 'eod_cookie': '0000_000000000_0000170', 'capacity_bytes': 858993459200L, 'non_del_files': 169, 'sum_wr_err': 0, 'sum_wr_access': 169, 'file_family': 'cms11', 'address': ('131.225.13.59', 7502), 'volume_family': 'cms.cms11.cpio_odc', 'modification_time': 1257314339.0, 'write_protected': 'n', 'sum_rd_err': 0, 'first_access': 1257307290.0}, 'outfilepath': '/pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v1/0080/46461190-BAC8-DE11-8522-00261894390C.root', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 25, 'delayed_dismount': None, 'curpri': 25}, 'file_size': 5832559230L, 'mover': {'max_buffer': 2621440000L, 'check_written_file': 600, 'max_failures': 5, 'library': 'CD-LTO4G1.library_manager', 'local_mover': 0, 'single_filemark': 1, 'max_consecutive_failures': 3, 'norestart': 'INQ', 'port': 7856, 'mc_device': '0,7,1,14', 'compression': 0, 'serial_num': '1310155266', 'update_interval_in_bound': 2, 'update_interval': 5, 'syslog_entry': 'sense|st[0-9]', 'status': ('ok', None), 'max_rate': 73400320.0, 'vendor_id': 'IBM', 'driver': 'FTTDriver', 'host': 'stkenmvr195a', 'max_time_in_state': 1200, 'device': 'stkenmvr195a:/dev/rmt/tps0d0n', 'name': 'LTO4_95.mover', 'send_stats': 1, 'product_id': 'ULTRIUM-TD4', 'statistics_path': '/tmp/enstore/enstore/LTO4_95.stat', 'mover_address': ('131.225.13.190', 7856), 'mount_delay': 15, 'check_first_written_file': 0, 'callback_addr': ('131.225.13.190', 41177), 'data_ip': 'stkenmvr195a', 'logname': 'LTO4_95MV', 'hostip': '131.225.13.190', 'do_cleaning': 'No', 'media_changer': 'SL8500G1.media_changer'}, 'ignore_fair_share': None, 'retry': None, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v1/0080/46461190-BAC8-DE11-8522-00261894390C.root', 'uid': 0, 'uname': 'root', 'mtime': 1257312708, 'rmajor': 0, 'machine': ('Linux', 'cmsstor208.fnal.gov', '2.6.31', '#1 SMP Mon Sep 14 09:43:20 CDT 2009', 'x86_64'), 'sanity_size': 65536, 'type': 'cpio_odc', 'pstat': (16893, 120195656, 21L, 1, 9811, 5063, 512, 1257293777, 1257293777, 1257279870), 'mode_octal': '0100036', 'mode': 33188, 'size_bytes': 5832559230L, 'gname': 'root', 'gid': 0, 'fullname': '/storage/data3/write-pool-3/data/0007000000000000622F5560', 'inode': 120542560L, 'minor': 21}, 'version': 'v3_7d CVS $Revision: 1.922 $ encp', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/storage/data3/write-pool-3/data/0007000000000000622F5560', 'status': ('ok', None), 'outfile': '/pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v1/0080/.(access)(0007000000000000622F5560)', 'fc': {'uid': 0, 'external_label': 'VOR957', 'gid': 0, 'location_cookie': '0000_000000000_0000170', 'complete_crc': 3986614447L, 'bfid': 'CDMS125731440400000', 'sanity_cookie': (65536L, 1824181559L), 'size': 5832559230L}, 'infilepath': '/storage/data3/write-pool-3/data/0007000000000000622F5560', 'at_the_top': 8, 'local_inode': 6400, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.191.79', 59266), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'encp_start_time': 1257312707.358331, 'seek_time': 0.02936100959777832, 'transfer_time': 64.367226839065552, 't0': 1257312707, 'mount_time': 0.014204025268554688, 'lm_dequeued': 1257314340.0809679, 'job_queued': 1257312708.357348, 'drive_call_time': 18.843962430953979, 'in_queue': 1631.7233221530914, 'drive_transfer_time': 50.194148063659668}, 'user_level': 'admin', 'unique_id': 'cmsstor208.fnal.gov-1257312708-23316-0'} Thread tape_thread 00:00:04 stkenmvr195a.fnal.gov 021479 root I LTO4_95MV Done is sent Thread tape_thread 00:00:04 stkenmvr195a.fnal.gov 021479 root I LTO4_95MV maybe_clean Thread tape_thread 00:00:04 cmsstor208.fnal.gov 023316 root I ENCP Received final dialog for cmsstor208.fnal.gov-1257312708-23316-0. elapsed=1697.230sec 00:00:04 stkenmvr195a.fnal.gov 021479 root I LTO4_95MV returned from maybe_clean Thread tape_thread 00:00:04 cmsstor208.fnal.gov 023316 root I ENCP leaving transfer_file(). 00:00:04 cmsstor208.fnal.gov 023316 root I ENCP pf_status = ('ok', None) skip_layer_cleanup = False 00:00:04 stkensrv0n.fnal.gov 028796 enstore I FILSRV Updating update file set deleted = 'n', pnfs_path = '/pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v1/0080/46461190-BAC8-DE11-8522-00261894390C.root', drive = 'stkenmvr195a:/dev/rmt/tps0d0n:1310155266', pnfs_id = '0007000000000000622F5560' where bfid = 'CDMS125731440400000'; 00:00:04 cmsstor208.fnal.gov 023316 root I ENCP MSG_TYPE=ENCP_XFER write_to_hsm /storage/data3/write-pool-3/data/0007000000000000622F5560 -> /pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v1/0080/46461190-BAC8-DE11-8522-00261894390C.root: 5832559230 bytes copied to VOR957 at 86.5 MB/S (94.9 MB/S network) (111 MB/S drive) (112 MB/S disk) mover=LTO4_95.mover drive_id=ULTRIUM-TD4 drive_sn=1310155266 drive_vendor=IBM elapsed=1697.6 {'storage_group': 'cms', 'encp_crc': 3986614447L, 'disk_rate': 112.08526199418911, 'drive_rate': 110.81694084168227, 'driver': 'FTTDriver', 'transfer_rate': 86.45560332141045, 'mover_interface': 'stkenmvr195a', 'overall_rate': 85.751231638717769, 'media_changer': 'SL8500G1.media_changer', 'network_rate': 86.45560332141045, 'encp_ip': '131.225.191.79', 'unique_id': 'cmsstor208.fnal.gov-1257312708-23316-0'} 00:00:05 cmsstor208.fnal.gov 023316 root I ENCP File /pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v1/0080/46461190-BAC8-DE11-8522-00261894390C.root copied successfully. 00:00:05 cmsstor208.fnal.gov 023316 root I ENCP Completed transferring 5832559230 bytes in 1 files in 1697.62510705 sec. Overall rate = 85.8 MB/sec. Transfer rate = 86.5 MB/sec. Network rate = 94.9 MB/sec. Drive rate = 111 MB/sec. Disk rate = 112 MB/sec. Exit status = 0. 00:00:05 cdfenmvr192a.fnal.gov 006151 root I LTO4_92MV written bytes 9447936/9447936, blocks 73 header 0 trailer 136 Thread tape_thread 00:00:05 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:05 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM write_to_hsm next work on vol=VOR957 mover=LTO4_95.mover requester:root 00:00:05 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM HAVE_BOUND:sending write_to_hsm /pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0080/58AAEAC1-BBC8-DE11-93CC-001731A28461.root to mover LTO4_95.mover ('131.225.13.190', 7856) DEL_DISM None 00:00:05 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM MSG_TYPE=ADD_TO_LMQ request added to work at movers queue for storage group : cms 00:00:05 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM HAVE_BOUND:sending {'vc': {'storage_group': 'cms', 'external_label': 'VOR957', 'library': 'CD-LTO4G1', 'file_family': 'cms11', 'wrapper': 'cpio_odc', 'address': ('131.225.13.59', 7502), 'volume_family': 'cms.cms11.cpio_odc', 'file_family_width': 10}, 'outfilepath': '/pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0080/58AAEAC1-BBC8-DE11-93CC-001731A28461.root', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 25, 'delayed_dismount': None, 'curpri': 25}, 'file_size': 2348017013L, 'mover': 'LTO4_95.mover', 'ignore_fair_share': None, 'retry': None, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0080/58AAEAC1-BBC8-DE11-93CC-001731A28461.root', 'uid': 0, 'uname': 'root', 'type': 'cpio_odc', 'gid': 0, 'mtime': 1257312799, 'rmajor': 0, 'machine': ('Linux', 'cmsstor206.fnal.gov', '2.6.31', '#1 SMP Mon Sep 14 09:43:20 CDT 2009', 'x86_64'), 'sanity_size': 65536, 'pstat': (16893, 120129424, 21L, 1, 9811, 5063, 512, 1257290009, 1257290009, 1257279324), 'mode_octal': '0100036', 'mode': 33188, 'gname': 'root', 'size_bytes': 2348017013L, 'fullname': '/storage/data2/write-pool-2/data/0007000000000000622F1F78', 'inode': 120528760L, 'minor': 21}, 'version': 'v3_7d CVS $Revision: 1.922 $ encp', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/storage/data2/write-pool-2/data/0007000000000000622F1F78', 'status': ('ok', None), 'outfile': '/pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0080/.(access)(0007000000000000622F1F78)', 'fc': {'size': 2348017013L, 'external_label': 'VOR957', 'address': ('131.225.13.59', 7501)}, 'infilepath': '/storage/data2/write-pool-2/data/0007000000000000622F1F78', 'at_the_top': 16, 'local_inode': 6839, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.191.77', 49362), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'lm_dequeued': 1257314405.2625401, 'encp_start_time': 1257312798.982126, 'job_queued': 1257312799.8738201, 't0': 1257312798, 'in_queue': 1605.3883938789368}, 'user_level': 'admin', 'unique_id': 'cmsstor206.fnal.gov-1257312799-1751-0'} to mover 00:00:05 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:05 stkenmvr195a.fnal.gov 021479 root I LTO4_95MV WRITE_TO_HSM Thread MainThread 00:00:05 stkenmvr195a.fnal.gov 021479 root I LTO4_95MV SENDING {'vc': {'storage_group': 'cms', 'external_label': 'VOR957', 'library': 'CD-LTO4G1', 'file_family': 'cms11', 'wrapper': 'cpio_odc', 'address': ('131.225.13.59', 7502), 'volume_family': 'cms.cms11.cpio_odc', 'file_family_width': 10}, 'outfilepath': '/pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0080/58AAEAC1-BBC8-DE11-93CC-001731A28461.root', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 25, 'delayed_dismount': None, 'curpri': 25}, 'file_size': 2348017013L, 'mover': {'max_buffer': 2621440000L, 'check_written_file': 600, 'max_failures': 5, 'library': 'CD-LTO4G1.library_manager', 'local_mover': 0, 'single_filemark': 1, 'max_consecutive_failures': 3, 'norestart': 'INQ', 'port': 7856, 'mc_device': '0,7,1,14', 'compression': 0, 'serial_num': '1310155266', 'update_interval_in_bound': 2, 'update_interval': 5, 'syslog_entry': 'sense|st[0-9]', 'status': ('ok', None), 'max_rate': 73400320.0, 'vendor_id': 'IBM', 'driver': 'FTTDriver', 'host': 'stkenmvr195a', 'max_time_in_state': 1200, 'device': 'stkenmvr195a:/dev/rmt/tps0d0n', 'name': 'LTO4_95.mover', 'send_stats': 1, 'product_id': 'ULTRIUM-TD4', 'statistics_path': '/tmp/enstore/enstore/LTO4_95.stat', 'mover_address': ('131.225.13.190', 7856), 'mount_delay': 15, 'check_first_written_file': 0, 'callback_addr': ('131.225.13.190', 41180), 'data_ip': 'stkenmvr195a', 'logname': 'LTO4_95MV', 'hostip': '131.225.13.190', 'do_cleaning': 'No', 'media_changer': 'SL8500G1.media_changer'}, 'ignore_fair_share': None, 'retry': None, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0080/58AAEAC1-BBC8-DE11-93CC-001731A28461.root', 'uid': 0, 'uname': 'root', 'mtime': 1257312799, 'rmajor': 0, 'machine': ('Linux', 'cmsstor206.fnal.gov', '2.6.31', '#1 SMP Mon Sep 14 09:43:20 CDT 2009', 'x86_64'), 'sanity_size': 65536, 'type': 'cpio_odc', 'pstat': (16893, 120129424, 21L, 1, 9811, 5063, 512, 1257290009, 1257290009, 1257279324), 'mode_octal': '0100036', 'mode': 33188, 'size_bytes': 2348017013L, 'gname': 'root', 'gid': 0, 'fullname': '/storage/data2/write-pool-2/data/0007000000000000622F1F78', 'inode': 120528760L, 'minor': 21}, 'version': 'v3_7d CVS $Revision: 1.922 $ encp', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/storage/data2/write-pool-2/data/0007000000000000622F1F78', 'status': ('ok', None), 'outfile': '/pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0080/.(access)(0007000000000000622F1F78)', 'fc': {'size': 2348017013L, 'external_label': 'VOR957', 'address': ('131.225.13.59', 7501)}, 'infilepath': '/storage/data2/write-pool-2/data/0007000000000000622F1F78', 'at_the_top': 16, 'local_inode': 6839, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.191.77', 49362), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'lm_dequeued': 1257314405.2625401, 'encp_start_time': 1257312798.982126, 'job_queued': 1257312799.8738201, 't0': 1257312798, 'in_queue': 1605.3883938789368}, 'user_level': 'admin', 'unique_id': 'cmsstor206.fnal.gov-1257312799-1751-0'} Thread client_connect_thread 00:00:05 cmsstor206.fnal.gov 001751 root I ENCP Control socket ('131.225.191.77', 49362) is connected to ('131.225.13.190', 41181) for cmsstor206.fnal.gov-1257312799-1751-0. elapsed=1606.350sec 00:00:05 cmsstor206.fnal.gov 001751 root I ENCP Received callback ticket from mover LTO4_95.mover for transfer cmsstor206.fnal.gov-1257312799-1751-0. elapsed=1606.350sec 00:00:05 cmsstor206.fnal.gov 001751 root I ENCP Atempting to connect data socket to mover at ('131.225.13.190', 41180). 00:00:05 cmsstor206.fnal.gov 001751 root I ENCP Connecting data socket to mover (('131.225.13.190', 41180)) with interface 131.225.191.77. elapsed=1606.350sec 00:00:05 cmsstor206.fnal.gov 001751 root I ENCP Data socket ('131.225.191.77', 50589) is connected to ('131.225.13.190', 41180). elapsed=1606.350sec 00:00:05 cmsstor206.fnal.gov 001751 root I ENCP Opening local file. 00:00:05 stkenmvr195a.fnal.gov 021479 root I LTO4_95MV tape position: target 171 current 171 0 Thread seek_thread 00:00:05 stkenmvr195a.fnal.gov 021479 root I LTO4_95MV write_tape starting, bytes_to_write=2348017664 Thread tape_thread 00:00:05 stkenmvr195a.fnal.gov 021479 root I LTO4_95MV Write starting. Tape VOR957 absolute location in blocks 3935347 Thread tape_thread 00:00:05 cmsstor206.fnal.gov 001751 root I ENCP pf_status = ('ok', None) skip_layer_cleanup = False 00:00:05 stkenmvr193a.fnal.gov 031647 root I LTO4_93MV filemarks written. Tape VON732 absolute location in blocks 3385013 Thread tape_thread 00:00:05 stkenmvr193a.fnal.gov 031647 root I LTO4_93MV write_tape timing:pull 0.218593835831 write 7.36852860451 crc_check 9.02600359917 freespace 0.143543720245 block_write 16.7566697598 idle 0.265034675598 Thread tape_thread 00:00:05 stkenmvr193a.fnal.gov 031647 root I LTO4_93MV drive stats after write. Tape VON732 position 234 block 0 block_size 0 bloc_loc 3385013 tot_blocks 762939000 BOT 0 read_err 0 write_err 0 bytes 1966080512 block_write_tot 16.7566697598 tape_rate 117331220.355 Thread tape_thread 00:00:05 stkenmvr193a.fnal.gov 031647 root I LTO4_93MV remaining bytes info in DB 363307008000 estimated 361340927488 reported from drive 361459712000 position 235 Thread tape_thread 00:00:05 stkenmvr193a.fnal.gov 031647 root I LTO4_93MV new bitfile request {'uid': 0, 'external_label': 'VON732', 'gid': 0, 'location_cookie': '0000_000000000_0000234', 'complete_crc': 79424410L, 'sanity_cookie': (65536L, 1725239357L), 'size': 1966080000L} Thread tape_thread 00:00:05 stkenmvr193a.fnal.gov 031647 root I LTO4_93MV set remaining: VON732 361459712000 0000_000000000_0000235 Thread tape_thread 00:00:05 last message repeated 1 times 00:00:05 stkensrv0n.fnal.gov 028762 enstore I VOLSRV Updating update volume set last_access = '2009-11-04 00:00:05', eod_cookie = '0000_000000000_0000235', non_del_files = '239', remaining_bytes = 361459712000 where label = 'VON732'; 00:00:06 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM 00:00:06 stkenmvr193a.fnal.gov 031647 root I LTO4_93MV transfer complete volume=VON732 location=235 Thread tape_thread 00:00:06 last message repeated 1 times 00:00:06 stkensrv0n.fnal.gov 028762 enstore I VOLSRV Updating update volume set sum_wr_access = '234', last_access = '2009-11-04 00:00:06', modification_time = '2009-11-04 00:00:06' where label = 'VON732'; 00:00:06 stkenmvr193a.fnal.gov 031647 root I LTO4_93MV Sending done to client: {'vc': {'comment': '', 'storage_group': 'exp-db', 'blocksize': 131072, 'declared': 1256853170.0, 'library': 'CD-LTO4G1', 'si_time': [0.0, 1255427870.0], 'user_inhibit': ['none', 'none'], 'system_inhibit': ['none', 'none'], 'external_label': 'VON732', 'wrapper': 'cpio_odc', 'remaining_bytes': 363307008000L, 'sum_mounts': 45, 'sum_rd_access': 0, 'file_family_width': 2, 'last_access': 1257314335.0, 'media_type': 'LTO4', 'status': ('ok', None), 'eod_cookie': '0000_000000000_0000234', 'capacity_bytes': 858993459200L, 'non_del_files': 238, 'sum_wr_err': 0, 'sum_wr_access': 233, 'file_family': 'daily-d0-offline', 'address': ('131.225.13.59', 7502), 'volume_family': 'exp-db.daily-d0-offline.cpio_odc', 'modification_time': 1257314335.0, 'write_protected': 'n', 'sum_rd_err': 0, 'first_access': 1257262991.0}, 'outfilepath': '/pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/03/channel01/D0OFPRD1_t701981422_s103228_p6', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 1, 'delayed_dismount': None, 'curpri': 1}, 'file_size': 1966080000L, 'mover': {'max_buffer': 2621440000L, 'check_written_file': 600, 'max_failures': 5, 'library': 'CD-LTO4G1.library_manager', 'local_mover': 0, 'single_filemark': 1, 'max_consecutive_failures': 3, 'norestart': 'INQ', 'port': 7854, 'mc_device': '0,7,1,11', 'compression': 0, 'serial_num': '9310097076', 'update_interval_in_bound': 2, 'update_interval': 5, 'syslog_entry': 'sense|st[0-9]', 'status': ('ok', None), 'max_rate': 73400320.0, 'vendor_id': 'IBM', 'driver': 'FTTDriver', 'host': 'stkenmvr193a', 'max_time_in_state': 1200, 'device': 'stkenmvr193a:/dev/rmt/tps0d0n', 'name': 'LTO4_93.mover', 'send_stats': 1, 'product_id': 'ULTRIUM-TD4', 'statistics_path': '/tmp/enstore/enstore/LTO4_93.stat', 'mover_address': ('131.225.13.188', 7854), 'mount_delay': 15, 'check_first_written_file': 0, 'callback_addr': ('131.225.13.188', 36316), 'data_ip': 'stkenmvr193a', 'logname': 'LTO4_93MV', 'hostip': '131.225.13.188', 'do_cleaning': 'No', 'media_changer': 'SL8500G1.media_changer'}, 'ignore_fair_share': None, 'retry': None, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/03/channel01/D0OFPRD1_t701981422_s103228_p6', 'uid': 0, 'uname': 'root', 'mtime': 1257314281, 'rmajor': 0, 'machine': ('Linux', 'stkendca18a.fnal.gov', '2.6.9-89.0.9.ELsmp', '#1 SMP Mon Aug 24 13:47:43 CDT 2009', 'i686'), 'sanity_size': 65536, 'type': 'cpio_odc', 'pstat': (16877, 820618152L, 20L, 1, 1602, 2752, 512L, 1257314271, 1257314271, 1254398418), 'mode_octal': '0100016', 'mode': 33188, 'size_bytes': 1966080000L, 'gname': 'root', 'gid': 0, 'fullname': '/diskb/write-pool-2/data/0030000000000000061D1020', 'inode': 807211040L, 'minor': 20}, 'version': 'v3_8b CVS $Revision: 1.922 $ ', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/diskb/write-pool-2/data/0030000000000000061D1020', 'status': ('ok', None), 'outfile': '/pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/03/channel01/.(access)(0030000000000000061D1020)', 'fc': {'uid': 0, 'external_label': 'VON732', 'gid': 0, 'location_cookie': '0000_000000000_0000234', 'complete_crc': 79424410L, 'bfid': 'CDMS125731440500000', 'sanity_cookie': (65536L, 1725239357L), 'size': 1966080000L}, 'infilepath': '/diskb/write-pool-2/data/0030000000000000061D1020', 'at_the_top': 82, 'local_inode': 609973993L, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.13.6', 49474), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'encp_start_time': 1257314280.1864829, 'seek_time': 0.030128002166748047, 'transfer_time': 70.186601877212524, 't0': 1257314280, 'mount_time': 0.014213085174560547, 'lm_dequeued': 1257314335.7317109, 'job_queued': 1257314281.2715869, 'drive_call_time': 7.1676394939422607, 'in_queue': 54.460020065307617, 'drive_transfer_time': 18.781302690505981}, 'user_level': 'user2', 'unique_id': 'stkendca18a.fnal.gov-1257314281-28506-0'} Thread tape_thread 00:00:06 stkenmvr193a.fnal.gov 031647 root I LTO4_93MV Done is sent Thread tape_thread 00:00:06 stkenmvr193a.fnal.gov 031647 root I LTO4_93MV maybe_clean Thread tape_thread 00:00:06 stkenmvr193a.fnal.gov 031647 root I LTO4_93MV returned from maybe_clean Thread tape_thread 00:00:06 stkendca18a.fnal.gov 028506 root I ENCP Received final dialog for stkendca18a.fnal.gov-1257314281-28506-0. elapsed=125.870sec 00:00:06 stkendca18a.fnal.gov 028506 root I ENCP leaving transfer_file(). 00:00:06 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:06 stkendca18a.fnal.gov 028506 root I ENCP pf_status = ('ok', None) skip_layer_cleanup = False 00:00:06 stkensrv0n.fnal.gov 028796 enstore I FILSRV Updating update file set deleted = 'n', pnfs_path = '/pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/03/channel01/D0OFPRD1_t701981422_s103228_p6', drive = 'stkenmvr193a:/dev/rmt/tps0d0n:9310097076', pnfs_id = '0030000000000000061D1020' where bfid = 'CDMS125731440500000'; 00:00:06 stkendca18a.fnal.gov 028506 root I ENCP MSG_TYPE=ENCP_XFER write_to_hsm /diskb/write-pool-2/data/0030000000000000061D1020 -> /pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/03/channel01/D0OFPRD1_t701981422_s103228_p6: 1966080000 bytes copied to VON732 at 26.7 MB/S (76 MB/S network) (99.8 MB/S drive) (31.2 MB/S disk) mover=LTO4_93.mover drive_id=ULTRIUM-TD4 drive_sn=9310097076 drive_vendor=IBM elapsed=126.23 {'storage_group': 'exp-db', 'encp_crc': 79424410L, 'disk_rate': 31.168348796137597, 'drive_rate': 99.833330568055828, 'driver': 'FTTDriver', 'transfer_rate': 26.743173514823678, 'mover_interface': 'stkenmvr193a', 'overall_rate': 26.546161717916124, 'media_changer': 'SL8500G1.media_changer', 'network_rate': 26.743173514823678, 'encp_ip': '131.225.13.6', 'unique_id': 'stkendca18a.fnal.gov-1257314281-28506-0'} 00:00:06 stkendca18a.fnal.gov 028506 root I ENCP File /pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/03/channel01/D0OFPRD1_t701981422_s103228_p6 copied successfully. 00:00:06 stkendca18a.fnal.gov 028506 root I ENCP Completed transferring 1966080000 bytes in 1 files in 126.229776144 sec. Overall rate = 26.5 MB/sec. Transfer rate = 26.7 MB/sec. Network rate = 76 MB/sec. Drive rate = 99.8 MB/sec. Disk rate = 31.2 MB/sec. Exit status = 0. 00:00:06 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:06 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:06 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM write_to_hsm next work on vol=VON732 mover=LTO4_93.mover requester:root 00:00:06 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM HAVE_BOUND:sending write_to_hsm /pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/03/channel02/D0OFPRD1_t701979310_s103226_p22 to mover LTO4_93.mover ('131.225.13.188', 7854) DEL_DISM None 00:00:06 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM MSG_TYPE=ADD_TO_LMQ request added to work at movers queue for storage group : exp-db 00:00:06 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM HAVE_BOUND:sending {'vc': {'storage_group': 'exp-db', 'external_label': 'VON732', 'library': 'CD-LTO4G1', 'file_family': 'daily-d0-offline', 'wrapper': 'cpio_odc', 'address': ('131.225.13.59', 7502), 'volume_family': 'exp-db.daily-d0-offline.cpio_odc', 'file_family_width': 2}, 'outfilepath': '/pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/03/channel02/D0OFPRD1_t701979310_s103226_p22', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 1, 'delayed_dismount': None, 'curpri': 1}, 'file_size': 1966080000L, 'mover': 'LTO4_93.mover', 'ignore_fair_share': None, 'retry': None, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/03/channel02/D0OFPRD1_t701979310_s103226_p22', 'uid': 0, 'uname': 'root', 'type': 'cpio_odc', 'gid': 0, 'mtime': 1257314337, 'rmajor': 0, 'machine': ('Linux', 'stkendca17a.fnal.gov', '2.6.9-89.0.9.ELsmp', '#1 SMP Mon Aug 24 13:47:43 CDT 2009', 'i686'), 'sanity_size': 65536, 'pstat': (16877, 820618368L, 20L, 1, 1602, 2752, 512L, 1257314296, 1257314296, 1254398418), 'mode_octal': '0100016', 'mode': 33188, 'gname': 'root', 'size_bytes': 1966080000L, 'fullname': '/diskc/write-pool-3/data/0030000000000000061D1078', 'inode': 807211128L, 'minor': 20}, 'version': 'v3_8b CVS $Revision: 1.922 $ ', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/diskc/write-pool-3/data/0030000000000000061D1078', 'status': ('ok', None), 'outfile': '/pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/03/channel02/.(access)(0030000000000000061D1078)', 'fc': {'size': 1966080000L, 'external_label': 'VON732', 'address': ('131.225.13.59', 7501)}, 'infilepath': '/diskc/write-pool-3/data/0030000000000000061D1078', 'at_the_top': 138, 'local_inode': 268645760L, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.13.5', 39502), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'lm_dequeued': 1257314406.824646, 'encp_start_time': 1257314336.353235, 'job_queued': 1257314337.2924671, 't0': 1257314336, 'in_queue': 69.532097816467285}, 'user_level': 'user2', 'unique_id': 'stkendca17a.fnal.gov-1257314337-25992-0'} to mover 00:00:06 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:06 stkenmvr193a.fnal.gov 031647 root I LTO4_93MV WRITE_TO_HSM Thread MainThread 00:00:06 stkenmvr193a.fnal.gov 031647 root I LTO4_93MV SENDING {'vc': {'storage_group': 'exp-db', 'external_label': 'VON732', 'library': 'CD-LTO4G1', 'file_family': 'daily-d0-offline', 'wrapper': 'cpio_odc', 'address': ('131.225.13.59', 7502), 'volume_family': 'exp-db.daily-d0-offline.cpio_odc', 'file_family_width': 2}, 'outfilepath': '/pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/03/channel02/D0OFPRD1_t701979310_s103226_p22', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 1, 'delayed_dismount': None, 'curpri': 1}, 'file_size': 1966080000L, 'mover': {'max_buffer': 2621440000L, 'check_written_file': 600, 'max_failures': 5, 'library': 'CD-LTO4G1.library_manager', 'local_mover': 0, 'single_filemark': 1, 'max_consecutive_failures': 3, 'norestart': 'INQ', 'port': 7854, 'mc_device': '0,7,1,11', 'compression': 0, 'serial_num': '9310097076', 'update_interval_in_bound': 2, 'update_interval': 5, 'syslog_entry': 'sense|st[0-9]', 'status': ('ok', None), 'max_rate': 73400320.0, 'vendor_id': 'IBM', 'driver': 'FTTDriver', 'host': 'stkenmvr193a', 'max_time_in_state': 1200, 'device': 'stkenmvr193a:/dev/rmt/tps0d0n', 'name': 'LTO4_93.mover', 'send_stats': 1, 'product_id': 'ULTRIUM-TD4', 'statistics_path': '/tmp/enstore/enstore/LTO4_93.stat', 'mover_address': ('131.225.13.188', 7854), 'mount_delay': 15, 'check_first_written_file': 0, 'callback_addr': ('131.225.13.188', 36320), 'data_ip': 'stkenmvr193a', 'logname': 'LTO4_93MV', 'hostip': '131.225.13.188', 'do_cleaning': 'No', 'media_changer': 'SL8500G1.media_changer'}, 'ignore_fair_share': None, 'retry': None, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/03/channel02/D0OFPRD1_t701979310_s103226_p22', 'uid': 0, 'uname': 'root', 'mtime': 1257314337, 'rmajor': 0, 'machine': ('Linux', 'stkendca17a.fnal.gov', '2.6.9-89.0.9.ELsmp', '#1 SMP Mon Aug 24 13:47:43 CDT 2009', 'i686'), 'sanity_size': 65536, 'type': 'cpio_odc', 'pstat': (16877, 820618368L, 20L, 1, 1602, 2752, 512L, 1257314296, 1257314296, 1254398418), 'mode_octal': '0100016', 'mode': 33188, 'size_bytes': 1966080000L, 'gname': 'root', 'gid': 0, 'fullname': '/diskc/write-pool-3/data/0030000000000000061D1078', 'inode': 807211128L, 'minor': 20}, 'version': 'v3_8b CVS $Revision: 1.922 $ ', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/diskc/write-pool-3/data/0030000000000000061D1078', 'status': ('ok', None), 'outfile': '/pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/03/channel02/.(access)(0030000000000000061D1078)', 'fc': {'size': 1966080000L, 'external_label': 'VON732', 'address': ('131.225.13.59', 7501)}, 'infilepath': '/diskc/write-pool-3/data/0030000000000000061D1078', 'at_the_top': 138, 'local_inode': 268645760L, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.13.5', 39502), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'lm_dequeued': 1257314406.824646, 'encp_start_time': 1257314336.353235, 'job_queued': 1257314337.2924671, 't0': 1257314336, 'in_queue': 69.532097816467285}, 'user_level': 'user2', 'unique_id': 'stkendca17a.fnal.gov-1257314337-25992-0'} Thread client_connect_thread 00:00:06 stkendca17a.fnal.gov 025992 root I ENCP Control socket ('131.225.13.5', 39502) is connected to ('131.225.13.188', 36321) for stkendca17a.fnal.gov-1257314337-25992-0. elapsed=70.530sec 00:00:06 stkendca17a.fnal.gov 025992 root I ENCP Received callback ticket from mover LTO4_93.mover for transfer stkendca17a.fnal.gov-1257314337-25992-0. elapsed=70.540sec 00:00:06 stkendca17a.fnal.gov 025992 root I ENCP Atempting to connect data socket to mover at ('131.225.13.188', 36320). 00:00:07 stkendca17a.fnal.gov 025992 root I ENCP Connecting data socket to mover (('131.225.13.188', 36320)) with interface 131.225.13.5. elapsed=70.540sec 00:00:07 stkendca17a.fnal.gov 025992 root I ENCP Data socket ('131.225.13.5', 39505) is connected to ('131.225.13.188', 36320). elapsed=70.540sec 00:00:07 stkendca17a.fnal.gov 025992 root I ENCP Opening local file. 00:00:07 stkenmvr193a.fnal.gov 031647 root I LTO4_93MV tape position: target 235 current 235 0 Thread seek_thread 00:00:07 stkenmvr193a.fnal.gov 031647 root I LTO4_93MV write_tape starting, bytes_to_write=1966080512 Thread tape_thread 00:00:07 stkendca17a.fnal.gov 025992 root I ENCP pf_status = ('ok', None) skip_layer_cleanup = False 00:00:07 stkenmvr193a.fnal.gov 031647 root I LTO4_93MV Write starting. Tape VON732 absolute location in blocks 3385013 Thread tape_thread 00:00:07 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:08 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM 00:00:08 stkendca18a.fnal.gov 029328 root I ENCP Version: v3_8b CVS $Revision: 1.922 $ OS: Linux 2.6.9-89.0.9.ELsmp i686 Release: Scientific Linux Fermi LTS release 4.4 (Wilson) User: root(0) Group: root(0) Euser: root(0) Egroup: root(0) Library: CD-LTO4G1 Storage Group: exp-db File Family: daily-d0-offline FF Wrapper: cpio_odc FF Width: 2 Current working directory: stkendca18a.fnal.gov:/home/enstore/dcache-code/dcache-fermi-config Command line: /fnal/ups/prd/encp/v3_8b/Linux-2-6-dcache/encp --verbose 4 --threaded --ecrc --bypass-filesystem-max-filesize-check --pnfs-mount /pnfs/fs --shortcut --override-path /pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/03/channel01/D0OFPRD1_t701984821_s103234_p14 --put-cache 0030000000000000061D10A0 /diska/write-pool-1/data/0030000000000000061D10A0 00:00:08 stkensrv0n.fnal.gov 023312 enstore I INFSRV start listing all volumes (3) 00:00:08 stkendca18a.fnal.gov 029328 root I ENCP Creating zero length output files. 00:00:08 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:08 stkendca18a.fnal.gov 029328 root I ENCP Sending /pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/03/channel01/D0OFPRD1_t701984821_s103234_p14 write request to LM: unique_id: stkendca18a.fnal.gov-1257314408-29328-0 inputfile: /diska/write-pool-1/data/0030000000000000061D10A0 outputfile: /pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/03/channel01/.(access)(0030000000000000061D10A0) 00:00:08 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM MSG_TYPE=ADD_TO_LMQ request added to pending queue for storage group : exp-db 00:00:08 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM write Q'd /diska/write-pool-1/data/0030000000000000061D10A0 (stkendca18a.fnal.gov-1257314408-29328-0) -> /pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/03/channel01/D0OFPRD1_t701984821_s103234_p14 : library=CD-LTO4G1 family=daily-d0-offline requester:root volume_family:exp-db.daily-d0-offline.cpio_odc 00:00:08 stkendca18a.fnal.gov 029328 root I ENCP Listening for control socket at: ('131.225.13.6', 49482) 00:00:08 stkendca18a.fnal.gov 029328 root I ENCP Waiting for mover to connect control socket at address: ('131.225.13.6', 49482) elapsed=1.210sec 00:00:09 stkenmvr197a.fnal.gov 025501 root I LTO4_97MV skip_fm 790 Thread seek_thread 00:00:09 stkenmvr197a.fnal.gov 025501 root I LTO4_97MV write_tape starting, bytes_to_write=1594721280 Thread tape_thread 00:00:09 stkenmvr197a.fnal.gov 025501 root I LTO4_97MV Write starting. Tape VON178 absolute location in blocks 4668136 Thread tape_thread 00:00:09 cdfenmvr192a.fnal.gov 006151 root I LTO4_92MV filemarks written. Tape VOV138 absolute location in blocks 255448 Thread tape_thread 00:00:09 cdfenmvr192a.fnal.gov 006151 root I LTO4_92MV write_tape timing:pull 0.000576734542847 write 0.00981140136719 crc_check 0.0430767536163 freespace 0.000566244125366 block_write 0.0540311336517 idle 0.00113940238953 Thread tape_thread 00:00:09 cdfenmvr192a.fnal.gov 006151 root I LTO4_92MV drive stats after write. Tape VOV138 position 5176 block 0 block_size 0 bloc_loc 255448 tot_blocks 762939000 BOT 0 read_err 0 write_err 14 bytes 9447936 block_write_tot 0.0540311336517 tape_rate 174860961.847 Thread tape_thread 00:00:09 cdfenmvr192a.fnal.gov 006151 root I LTO4_92MV remaining bytes info in DB 738942976000 estimated 738933528064 reported from drive 738928640000 position 5179 Thread tape_thread 00:00:09 cdfenmvr192a.fnal.gov 006151 root I LTO4_92MV new bitfile request {'uid': 0, 'external_label': 'VOV138', 'gid': 0, 'location_cookie': '0000_000000000_0005177', 'complete_crc': 1018318751L, 'sanity_cookie': (65536L, 3597226830L), 'size': 9447800L} Thread tape_thread 00:00:09 stkensrv4n.fnal.gov 003689 enstore I SL8500G1MC FINISHED listDrives returned ('ok', 0, None, '', '') 00:00:09 cdfenmvr192a.fnal.gov 006151 root I LTO4_92MV set remaining: VOV138 738928640000 0000_000000000_0005179 Thread tape_thread 00:00:09 last message repeated 1 times 00:00:09 stkensrv0n.fnal.gov 028762 enstore I VOLSRV Updating update volume set last_access = '2009-11-04 00:00:09', eod_cookie = '0000_000000000_0005179', non_del_files = '1727', remaining_bytes = 738928640000 where label = 'VOV138'; 00:00:09 cdfenmvr192a.fnal.gov 006151 root I LTO4_92MV transfer complete volume=VOV138 location=5179 Thread tape_thread 00:00:09 last message repeated 1 times 00:00:09 stkensrv0n.fnal.gov 028762 enstore I VOLSRV Updating update volume set sum_wr_access = '1726', modification_time = '2009-11-04 00:00:09' where label = 'VOV138'; 00:00:09 cdfenmvr192a.fnal.gov 006151 root I LTO4_92MV Sending done to client: {'vc': {'comment': '', 'storage_group': 'ktev', 'blocksize': 131072, 'declared': 1254165875.0, 'library': 'CD-LTO4F1', 'si_time': [0.0, 0.0], 'user_inhibit': ['none', 'none'], 'system_inhibit': ['none', 'none'], 'external_label': 'VOV138', 'wrapper': 'cern', 'remaining_bytes': 738942976000L, 'sum_mounts': 1, 'sum_rd_access': 0, 'file_family_width': 1, 'last_access': 1257314403.0, 'media_type': 'LTO4', 'status': ('ok', None), 'eod_cookie': '0000_000000000_0005176', 'capacity_bytes': 858993459200L, 'non_del_files': 1726, 'sum_wr_err': 0, 'sum_wr_access': 1725, 'file_family': 'ktev_migration-MIGRATION', 'address': ('131.225.13.59', 7502), 'volume_family': 'ktev.ktev_migration-MIGRATION.cern', 'modification_time': 1257314403.0, 'write_protected': 'n', 'sum_rd_err': 0, 'first_access': 1257302588.0}, 'outfilepath': '/pnfs/fs/usr/Migration/ktev/migrated_from_dlt/KQ/KQH0/KQH054/f336', 'encp': {'agetime': 30, 'adminpri': 6, 'delpri': 1, 'basepri': 1000007, 'delayed_dismount': 2, 'curpri': 1000007}, 'file_size': 9447800L, 'mover': {'max_buffer': 2621440000L, 'check_written_file': 600, 'max_failures': 5, 'library': 'CD-LTO4F1.library_manager', 'local_mover': 0, 'single_filemark': 1, 'max_consecutive_failures': 3, 'norestart': 'INQ', 'port': 7853, 'mc_device': '1,0,1,1', 'compression': 0, 'serial_num': '1310060800', 'update_interval_in_bound': 2, 'update_interval': 5, 'syslog_entry': 'sense|st[0-9]', 'status': ('ok', None), 'max_rate': 73400320.0, 'vendor_id': 'IBM', 'driver': 'FTTDriver', 'host': 'cdfenmvr192a', 'max_time_in_state': 1200, 'device': 'cdfenmvr192a:/dev/rmt/tps0d0n', 'name': 'LTO4_92.mover', 'send_stats': 1, 'product_id': 'ULTRIUM-TD4', 'statistics_path': '/tmp/enstore/enstore/LTO4_92.stat', 'mover_address': ('131.225.215.76', 7853), 'mount_delay': 15, 'check_first_written_file': 0, 'callback_addr': ('131.225.215.76', 60647), 'data_ip': 'cdfenmvr192a', 'logname': 'LTO4_92MV', 'hostip': '131.225.215.76', 'do_cleaning': 'No', 'media_changer': 'SL8500F1.media_changer'}, 'ignore_fair_share': 1, 'retry': None, 'lm': {'address': ('131.225.13.4', 8542)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fs/usr/ktev/migrated_from_dlt/KQ/KQH0/KQH054/f336', 'uid': 0, 'uname': 'root', 'mtime': 1257314391, 'rmajor': 0, 'machine': ('Linux', 'stkendm7a.fnal.gov', '2.6.9-89.0.9.ELsmp', '#1 SMP Mon Aug 24 13:47:43 CDT 2009', 'i686'), 'sanity_size': 65536, 'type': 'cern', 'pstat': (16877, 686516192L, 20L, 1, 0, 0, 512L, 1257313751, 1257313751, 1255446031), 'mode_octal': '0100016', 'mode': 33188, 'size_bytes': 9447800L, 'gname': 'root', 'gid': 0, 'fullname': '/data/data2/Migration_Spool/VO6153:0000_000000000_0031880', 'inode': 675389696L, 'minor': 20}, 'version': 'v3_8 CVS $Revision: 1.963 $ encp.pyc', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/data/data2/Migration_Spool/VO6153:0000_000000000_0031880', 'status': ('ok', None), 'outfile': '/pnfs/fs/usr/Migration/ktev/migrated_from_dlt/KQ/KQH0/KQH054/.(access)(00280000000000000841A100)', 'fc': {'uid': 0, 'external_label': 'VOV138', 'gid': 0, 'location_cookie': '0000_000000000_0005177', 'complete_crc': 1018318751L, 'bfid': 'CDMS125731440900000', 'sanity_cookie': (65536L, 3597226830L), 'size': 9447800L}, 'infilepath': '/data/data2/Migration_Spool/VO6153:0000_000000000_0031880', 'at_the_top': 36, 'local_inode': 33308833L, 'resend': {'max_retry': 3, 'resubmits': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.13.19', 57436), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'encp_start_time': 1257314391.778286, 'seek_time': 0.02682805061340332, 'transfer_time': 5.9396100044250488, 't0': 1257314391, 'mount_time': 0.016020059585571289, 'lm_dequeued': 1257314403.378454, 'job_queued': 1257314392.013905, 'drive_call_time': 0.019969940185546875, 'in_queue': 11.364289045333862, 'drive_transfer_time': 5.7669355869293213}, 'user_level': 'admin', 'unique_id': 'stkendm7a.fnal.gov-1257314391-18643-24630'} Thread tape_thread 00:00:09 cdfenmvr192a.fnal.gov 006151 root I LTO4_92MV Done is sent Thread tape_thread 00:00:09 cdfenmvr192a.fnal.gov 006151 root I LTO4_92MV maybe_clean Thread tape_thread 00:00:09 stkendm7a.fnal.gov 018643 root I ENCP Received final dialog for stkendm7a.fnal.gov-1257314391-18643-24630. elapsed=154655.850sec Thread WRITE 00:00:09 stkendm7a.fnal.gov 018643 root I ENCP leaving transfer_file(). Thread WRITE 00:00:09 cdfenmvr192a.fnal.gov 006151 root I LTO4_92MV returned from maybe_clean Thread tape_thread 00:00:09 stkensrv0n.fnal.gov 028796 enstore I FILSRV Updating update file set deleted = 'n', pnfs_path = '/pnfs/fs/usr/ktev/migrated_from_dlt/KQ/KQH0/KQH054/f336', drive = 'cdfenmvr192a:/dev/rmt/tps0d0n:1310060800', pnfs_id = '00280000000000000841A100' where bfid = 'CDMS125731440900000'; 00:00:09 cmsstor236.fnal.gov 025186 root I ENCP Version: v3_7d CVS $Revision: 1.922 $ encp OS: Linux 2.6.31 x86_64 Release: Rocks release 5.1 (V.I) Scientific Linux SL release 5.2 (Boron) User: root(0) Group: root(0) Euser: root(0) Egroup: root(0) Library: CD-LTO4G1 Storage Group: cms File Family: cms11 FF Wrapper: cpio_odc FF Width: 10 Current working directory: cmsstor236.fnal.gov:/root Command line: /home/enstore/enstore/bin/encp --verbose 4 --threaded --ecrc --resubmit-timeout 1800 --priority 25 --pnfs-mount /pnfs/fs --put-cache 000700000000000062404B48 /storage/data2/write-pool-2/data/000700000000000062404B48 00:00:09 cmsstor208.fnal.gov 000434 root I ENCP Version: v3_7d CVS $Revision: 1.922 $ encp OS: Linux 2.6.31 x86_64 Release: Rocks release 5.1 (V.I) Scientific Linux SL release 5.2 (Boron) User: root(0) Group: root(0) Euser: root(0) Egroup: root(0) Library: CD-LTO4G1 Storage Group: cms File Family: cms2 FF Wrapper: cpio_odc FF Width: 7 Current working directory: cmsstor208.fnal.gov:/ Command line: /home/enstore/enstore/bin/encp --verbose 4 --threaded --ecrc --resubmit-timeout 1800 --age-time 30 --delpri 1 --priority 25 --pnfs-mount /pnfs/fs --shortcut --override-path /pnfs/fnal.gov/usr/cms/WAX/2/nsonmez/pat_v3/Pt0to15/QCDDiJet_Pt0to15_patLayer1_fromAOD_full_v2_288_C382344.root --put-cache 0010000000000000094E7E80 /storage/data1/write-pool-1/data/0010000000000000094E7E80 00:00:09 stkendm7a.fnal.gov 018643 root I ENCP MSG_TYPE=ENCP_XFER write_to_hsm /data/data2/Migration_Spool/VO6153:0000_000000000_0031880 -> /pnfs/fs/usr/Migration/ktev/migrated_from_dlt/KQ/KQH0/KQH054/f336: 9447800 bytes copied to VOV138 at 1.52 MB/S (90.7 MB/S network) (1.56 MB/S drive) (102 MB/S disk) mover=LTO4_92.mover drive_id=ULTRIUM-TD4 drive_sn=1310060800 drive_vendor=IBM elapsed=18.009 {'storage_group': 'ktev', 'encp_crc': 1018318751L, 'disk_rate': 101.84844069542643, 'drive_rate': 1.5623764251787879, 'driver': 'FTTDriver', 'transfer_rate': 1.5153163991644092, 'mover_interface': 'cdfenmvr192a', 'overall_rate': 1.4135151621898234, 'media_changer': 'SL8500F1.media_changer', 'network_rate': 1.5153163991644092, 'encp_ip': '131.225.13.19', 'unique_id': 'stkendm7a.fnal.gov-1257314391-18643-24630'} Thread WRITE 00:00:09 stkendm7a.fnal.gov 018643 root I ENCP File /pnfs/fs/usr/Migration/ktev/migrated_from_dlt/KQ/KQH0/KQH054/f336 copied successfully. Thread WRITE 00:00:09 stkendm7a.fnal.gov 018643 root I ENCP Completed transferring 9447800 bytes in 1 files in 18.0106561184 sec. Overall rate = 1.41 MB/sec. Transfer rate = 1.52 MB/sec. Network rate = 90.7 MB/sec. Drive rate = 1.56 MB/sec. Disk rate = 102 MB/sec. Exit status = 0. Thread WRITE 00:00:10 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM 00:00:10 stkensrv0n.fnal.gov 028796 enstore I FILSRV start modifying {'pnfsid': '00280000000000000841A100', 'pnfs_name0': '/pnfs/fs/usr/ktev/migrated_from_dlt/KQ/KQH0/KQH054/f336', 'deleted': 'no', 'external_label': 'VOV138', 'drive': 'cdfenmvr192a:/dev/rmt/tps0d0n:1310060800', 'update': '2009-11-04 00:00:09.532786', 'gid': 0, 'location_cookie': '0000_000000000_0005177', 'complete_crc': 1018318751L, 'uid': 0, 'bfid': 'CDMS125731440900000', 'sanity_cookie': (65536L, 3597226830L), 'size': 9447800L} 00:00:10 stkensrv0n.fnal.gov 028796 enstore I FILSRV Updating update file set pnfs_id = '001F00000000000000DC8108' where bfid = 'CDMS125731440900000'; 00:00:10 stkensrv0n.fnal.gov 028796 enstore I FILSRV modified to {'pnfsid': '001F00000000000000DC8108', 'pnfs_name0': '/pnfs/fs/usr/ktev/migrated_from_dlt/KQ/KQH0/KQH054/f336', 'deleted': 'no', 'external_label': 'VOV138', 'drive': 'cdfenmvr192a:/dev/rmt/tps0d0n:1310060800', 'update': '2009-11-04 00:00:09.532786', 'gid': 0, 'location_cookie': '0000_000000000_0005177', 'complete_crc': 1018318751L, 'uid': 0, 'bfid': 'CDMS125731440900000', 'sanity_cookie': (65536L, 3597226830L), 'size': 9447800L} 00:00:10 cmsstor208.fnal.gov 000434 root I ENCP Creating zero length output files. 00:00:10 cmsstor236.fnal.gov 025186 root I ENCP Creating zero length output files. 00:00:10 cmsstor208.fnal.gov 000434 root I ENCP Sending /pnfs/fnal.gov/usr/cms/WAX/2/nsonmez/pat_v3/Pt0to15/QCDDiJet_Pt0to15_patLayer1_fromAOD_full_v2_288_C382344.root write request to LM: unique_id: cmsstor208.fnal.gov-1257314409-434-0 inputfile: /storage/data1/write-pool-1/data/0010000000000000094E7E80 outputfile: /pnfs/fnal.gov/usr/cms/WAX/2/nsonmez/pat_v3/Pt0to15/.(access)(0010000000000000094E7E80) 00:00:10 cmsstor236.fnal.gov 025186 root I ENCP Sending /pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0083/D290FDC8-D2C8-DE11-83B1-001A92971AA4.root write request to LM: unique_id: cmsstor236.fnal.gov-1257314409-25186-0 inputfile: /storage/data2/write-pool-2/data/000700000000000062404B48 outputfile: /pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0083/.(access)(000700000000000062404B48) 00:00:10 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM MSG_TYPE=ADD_TO_LMQ request added to pending queue for storage group : cms 00:00:10 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM write Q'd /storage/data1/write-pool-1/data/0010000000000000094E7E80 (cmsstor208.fnal.gov-1257314409-434-0) -> /pnfs/fnal.gov/usr/cms/WAX/2/nsonmez/pat_v3/Pt0to15/QCDDiJet_Pt0to15_patLayer1_fromAOD_full_v2_288_C382344.root : library=CD-LTO4G1 family=cms2 requester:root volume_family:cms.cms2.cpio_odc 00:00:10 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM MSG_TYPE=ADD_TO_LMQ request added to pending queue for storage group : cms 00:00:10 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM write Q'd /storage/data2/write-pool-2/data/000700000000000062404B48 (cmsstor236.fnal.gov-1257314409-25186-0) -> /pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0083/D290FDC8-D2C8-DE11-83B1-001A92971AA4.root : library=CD-LTO4G1 family=cms11 requester:root volume_family:cms.cms11.cpio_odc 00:00:10 cmsstor208.fnal.gov 000434 root I ENCP Listening for control socket at: ('131.225.191.79', 49601) 00:00:10 cmsstor208.fnal.gov 000434 root I ENCP Waiting for mover to connect control socket at address: ('131.225.191.79', 49601) elapsed=0.960sec 00:00:10 cmsstor236.fnal.gov 025186 root I ENCP Listening for control socket at: ('131.225.191.108', 52287) 00:00:10 cmsstor236.fnal.gov 025186 root I ENCP Waiting for mover to connect control socket at address: ('131.225.191.108', 52287) elapsed=1.150sec 00:00:10 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:10 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM 00:00:10 stkendm7a.fnal.gov 018643 root I ENCP Version: v3_8 CVS $Revision: 1.963 $ encp.pyc OS: Linux 2.6.9-89.0.9.ELsmp i686 Release: Scientific Linux Fermi LTS release 4.4 (Wilson) User: root(0) Group: root(0) Euser: root(0) Egroup: root(0) Library: CD-LTO4F1 Storage Group: ktev File Family: ktev_migration-MIGRATION FF Wrapper: cern FF Width: 1 Current working directory: stkendm7a.fnal.gov:/opt/enstore/src Command line: encp --delayed-dismount 2 --ignore-fair-share --threaded --priority 0 --storage-group ktev --file-family ktev_migration-MIGRATION --file-family-wrapper cern --library CD-LTO4F1 --override-path /pnfs/fs/usr/ktev/migrated_from_dlt/KQ/KQRC/KQRC44/f6813 --threaded /data/data2/Migration_Spool/VO6153:0000_000000000_0031889 /pnfs/fs/usr/Migration/ktev/migrated_from_dlt/KQ/KQRC/KQRC44/f6813 Thread WRITE 00:00:10 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM write_to_hsm next work on vol=VOV138 mover=LTO4_92.mover requester:root 00:00:10 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM HAVE_BOUND:sending write_to_hsm /pnfs/fs/usr/ktev/migrated_from_dlt/KQ/KQRC/KQRC38/f6698 to mover LTO4_92.mover ('131.225.215.76', 7853) DEL_DISM 2 00:00:10 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM MSG_TYPE=ADD_TO_LMQ request added to work at movers queue for storage group : ktev 00:00:10 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM HAVE_BOUND:sending {'vc': {'storage_group': 'ktev', 'external_label': 'VOV138', 'library': 'CD-LTO4F1', 'file_family': 'ktev_migration-MIGRATION', 'wrapper': 'cern', 'address': ('131.225.13.59', 7502), 'volume_family': 'ktev.ktev_migration-MIGRATION.cern', 'file_family_width': 1}, 'outfilepath': '/pnfs/fs/usr/Migration/ktev/migrated_from_dlt/KQ/KQRC/KQRC38/f6698', 'encp': {'agetime': 30, 'adminpri': 6, 'delpri': 1, 'basepri': 1000007, 'delayed_dismount': 2, 'curpri': 1000007}, 'file_size': 251340L, 'mover': 'LTO4_92.mover', 'ignore_fair_share': 1, 'retry': None, 'lm': {'address': ('131.225.13.4', 8542)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fs/usr/ktev/migrated_from_dlt/KQ/KQRC/KQRC38/f6698', 'uid': 0, 'uname': 'root', 'type': 'cern', 'gid': 0, 'mtime': 1257314397, 'rmajor': 0, 'machine': ('Linux', 'stkendm7a.fnal.gov', '2.6.9-89.0.9.ELsmp', '#1 SMP Mon Aug 24 13:47:43 CDT 2009', 'i686'), 'sanity_size': 65536, 'pstat': (16877, 678574944L, 20L, 1, 0, 0, 512L, 1257314008, 1257314008, 1252942702), 'mode_octal': '0100016', 'mode': 33188, 'gname': 'root', 'size_bytes': 251340L, 'fullname': '/data/data2/Migration_Spool/VO6153:0000_000000000_0032549', 'inode': 675389728L, 'minor': 20}, 'version': 'v3_8 CVS $Revision: 1.963 $ encp.pyc', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/data/data2/Migration_Spool/VO6153:0000_000000000_0032549', 'status': ('ok', None), 'outfile': '/pnfs/fs/usr/Migration/ktev/migrated_from_dlt/KQ/KQRC/KQRC38/.(access)(00280000000000000841A120)', 'fc': {'size': 251340L, 'external_label': 'VOV138', 'pnfsid': '00280000000000000841A120', 'address': ('131.225.13.59', 7501)}, 'infilepath': '/data/data2/Migration_Spool/VO6153:0000_000000000_0032549', 'at_the_top': 45, 'local_inode': 4495418L, 'resend': {'max_retry': 3, 'resubmits': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.13.19', 57439), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'lm_dequeued': 1257314410.4020441, 'encp_start_time': 1257314397.8037109, 'job_queued': 1257314398.007931, 't0': 1257314397, 'in_queue': 12.393855094909668}, 'user_level': 'admin', 'unique_id': 'stkendm7a.fnal.gov-1257314397-18643-24631'} to mover 00:00:10 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM 00:00:10 cdfenmvr192a.fnal.gov 006151 root I LTO4_92MV WRITE_TO_HSM Thread MainThread 00:00:10 cdfenmvr192a.fnal.gov 006151 root I LTO4_92MV SENDING {'vc': {'storage_group': 'ktev', 'external_label': 'VOV138', 'library': 'CD-LTO4F1', 'file_family': 'ktev_migration-MIGRATION', 'wrapper': 'cern', 'address': ('131.225.13.59', 7502), 'volume_family': 'ktev.ktev_migration-MIGRATION.cern', 'file_family_width': 1}, 'outfilepath': '/pnfs/fs/usr/Migration/ktev/migrated_from_dlt/KQ/KQRC/KQRC38/f6698', 'encp': {'agetime': 30, 'adminpri': 6, 'delpri': 1, 'basepri': 1000007, 'delayed_dismount': 2, 'curpri': 1000007}, 'file_size': 251340L, 'mover': {'max_buffer': 2621440000L, 'check_written_file': 600, 'max_failures': 5, 'library': 'CD-LTO4F1.library_manager', 'local_mover': 0, 'single_filemark': 1, 'max_consecutive_failures': 3, 'norestart': 'INQ', 'port': 7853, 'mc_device': '1,0,1,1', 'compression': 0, 'serial_num': '1310060800', 'update_interval_in_bound': 2, 'update_interval': 5, 'syslog_entry': 'sense|st[0-9]', 'status': ('ok', None), 'max_rate': 73400320.0, 'vendor_id': 'IBM', 'driver': 'FTTDriver', 'host': 'cdfenmvr192a', 'max_time_in_state': 1200, 'device': 'cdfenmvr192a:/dev/rmt/tps0d0n', 'name': 'LTO4_92.mover', 'send_stats': 1, 'product_id': 'ULTRIUM-TD4', 'statistics_path': '/tmp/enstore/enstore/LTO4_92.stat', 'mover_address': ('131.225.215.76', 7853), 'mount_delay': 15, 'check_first_written_file': 0, 'callback_addr': ('131.225.215.76', 60650), 'data_ip': 'cdfenmvr192a', 'logname': 'LTO4_92MV', 'hostip': '131.225.215.76', 'do_cleaning': 'No', 'media_changer': 'SL8500F1.media_changer'}, 'ignore_fair_share': 1, 'retry': None, 'lm': {'address': ('131.225.13.4', 8542)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fs/usr/ktev/migrated_from_dlt/KQ/KQRC/KQRC38/f6698', 'uid': 0, 'uname': 'root', 'mtime': 1257314397, 'rmajor': 0, 'machine': ('Linux', 'stkendm7a.fnal.gov', '2.6.9-89.0.9.ELsmp', '#1 SMP Mon Aug 24 13:47:43 CDT 2009', 'i686'), 'sanity_size': 65536, 'type': 'cern', 'pstat': (16877, 678574944L, 20L, 1, 0, 0, 512L, 1257314008, 1257314008, 1252942702), 'mode_octal': '0100016', 'mode': 33188, 'size_bytes': 251340L, 'gname': 'root', 'gid': 0, 'fullname': '/data/data2/Migration_Spool/VO6153:0000_000000000_0032549', 'inode': 675389728L, 'minor': 20}, 'version': 'v3_8 CVS $Revision: 1.963 $ encp.pyc', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/data/data2/Migration_Spool/VO6153:0000_000000000_0032549', 'status': ('ok', None), 'outfile': '/pnfs/fs/usr/Migration/ktev/migrated_from_dlt/KQ/KQRC/KQRC38/.(access)(00280000000000000841A120)', 'fc': {'size': 251340L, 'external_label': 'VOV138', 'pnfsid': '00280000000000000841A120', 'address': ('131.225.13.59', 7501)}, 'infilepath': '/data/data2/Migration_Spool/VO6153:0000_000000000_0032549', 'at_the_top': 45, 'local_inode': 4495418L, 'resend': {'max_retry': 3, 'resubmits': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.13.19', 57439), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'lm_dequeued': 1257314410.4020441, 'encp_start_time': 1257314397.8037109, 'job_queued': 1257314398.007931, 't0': 1257314397, 'in_queue': 12.393855094909668}, 'user_level': 'admin', 'unique_id': 'stkendm7a.fnal.gov-1257314397-18643-24631'} Thread client_connect_thread 00:00:10 stkendm7a.fnal.gov 018643 root I ENCP Listening for control socket at: ('131.225.13.19', 57439) Thread WRITE_1 00:00:10 stkendm7a.fnal.gov 018643 root I ENCP Waiting for mover to connect control socket at address: ('131.225.13.19', 57439) elapsed=154656.860sec Thread WRITE_1 00:00:10 stkendm7a.fnal.gov 018643 root I ENCP Control socket ('131.225.13.19', 57439) is connected to ('131.225.215.76', 60651) for stkendm7a.fnal.gov-1257314397-18643-24631. elapsed=154656.860sec Thread WRITE_1 00:00:10 stkendm7a.fnal.gov 018643 root I ENCP Received callback ticket from mover LTO4_92.mover for transfer stkendm7a.fnal.gov-1257314397-18643-24631. elapsed=154656.870sec Thread WRITE_1 00:00:10 stkendm7a.fnal.gov 018643 root I ENCP Attempting to connect data socket to mover at ('131.225.215.76', 60650). Thread WRITE_1 00:00:10 stkendm7a.fnal.gov 018643 root I ENCP Connecting data socket to mover (('131.225.215.76', 60650)) with interface 131.225.13.19. elapsed=154656.870sec Thread WRITE_1 00:00:10 stkendm7a.fnal.gov 018643 root I ENCP Data socket ('131.225.13.19', 57445) is connected to ('131.225.215.76', 60650). elapsed=154656.870sec Thread WRITE_1 00:00:10 stkendm7a.fnal.gov 018643 root I ENCP Opening local file. Thread WRITE_1 00:00:10 stkendm7a.fnal.gov 018643 root I ENCP Creating zero length output files. Thread WRITE 00:00:10 stkendm7a.fnal.gov 018643 root I ENCP pf_status = ('ok', None) skip_layer_cleanup = False Thread WRITE_1 00:00:10 cdfenmvr192a.fnal.gov 006151 root I LTO4_92MV tape position: target 5179 current 5179 0 Thread seek_thread 00:00:10 cdfenmvr192a.fnal.gov 006151 root I LTO4_92MV write_tape starting, bytes_to_write=251392 Thread tape_thread 00:00:10 stkendm7a.fnal.gov 018643 root I ENCP Sending /pnfs/fs/usr/Migration/ktev/migrated_from_dlt/KQ/KQRC/KQRC44/f6813 write request to LM: unique_id: stkendm7a.fnal.gov-1257314410-18643-24633 inputfile: /data/data2/Migration_Spool/VO6153:0000_000000000_0031889 outputfile: /pnfs/fs/usr/Migration/ktev/migrated_from_dlt/KQ/KQRC/KQRC44/.(access)(00280000000000000841A160) Thread WRITE 00:00:10 cdfenmvr192a.fnal.gov 006151 root I LTO4_92MV Write starting. Tape VOV138 absolute location in blocks 255448 Thread tape_thread 00:00:10 stkendm7a.fnal.gov 018643 root I ENCP File /data/data2/Migration_Spool/VO6153:0000_000000000_0032549 transfered for stkendm7a.fnal.gov-1257314397-18643-24631 in 0.00553512573242 seconds. Thread WRITE_1 00:00:10 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:10 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM MSG_TYPE=ADD_TO_LMQ request added to pending queue for storage group : ktev 00:00:10 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM write Q'd /data/data2/Migration_Spool/VO6153:0000_000000000_0031889 (stkendm7a.fnal.gov-1257314410-18643-24633) -> /pnfs/fs/usr/ktev/migrated_from_dlt/KQ/KQRC/KQRC44/f6813 : library=CD-LTO4F1 family=ktev_migration-MIGRATION requester:root volume_family:ktev.ktev_migration-MIGRATION.cern 00:00:11 stkensrv4n.fnal.gov 003747 enstore I SL8500G1MC mount VOJ149 0,6,1,8 => 0,Mount: VOJ149 mounted on 0, 6, 1, 8 00:00:11 stkensrv4n.fnal.gov 003689 enstore I SL8500G1MC FINISHED mount VOJ149 0,6,1,8 returned ('ok', 0, 'mount VOJ149 0,6,1,8 => 0,Mount: VOJ149 mounted on 0, 6, 1, 8', '', '') 00:00:11 last message repeated 1 times 00:00:11 stkensrv0n.fnal.gov 028762 enstore I VOLSRV Updating update volume set sum_mounts = '89', last_access = '2009-11-04 00:00:11' where label = 'VOJ149'; 00:00:11 stkenmvr206a.fnal.gov 026766 root I LTO4_106MV MSG_TYPE=MC_LOAD_DONE mounted VOJ149 ULTRIUM-TD4 00:00:11 Thread media_thread 00:00:11 stkensrv0n.fnal.gov 028762 enstore I VOLSRV Assigning volume VOS175 from storage group cms.cms2.cpio_odc to library CD-LTO4G1, volume family cms.cms2.cpio_odc 00:00:11 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM write_to_hsm work on vol=VOS175 mover=LTO4_48.mover requester:root 00:00:11 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM MSG_TYPE=ADD_TO_LMQ request added to work at movers queue for storage group : cms 00:00:11 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM IDLE:sending {'vc': {'storage_group': 'cms', 'library': 'CD-LTO4G1', 'file_family': 'cms2', 'wrapper': 'cpio_odc', 'address': ('131.225.13.59', 7502), 'volume_family': 'cms.cms2.cpio_odc', 'file_family_width': 7}, 'outfilepath': '/pnfs/fnal.gov/usr/cms/WAX/2/nsonmez/pat_v3/Pt0to15/QCDDiJet_Pt0to15_patLayer1_fromAOD_full_v2_288_C382344.root', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 25, 'delayed_dismount': None, 'curpri': 25}, 'file_size': 321859275L, 'mover': 'LTO4_48.mover', 'ignore_fair_share': None, 'retry': None, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fnal.gov/usr/cms/WAX/2/nsonmez/pat_v3/Pt0to15/QCDDiJet_Pt0to15_patLayer1_fromAOD_full_v2_288_C382344.root', 'uid': 0, 'uname': 'root', 'type': 'cpio_odc', 'gid': 0, 'mtime': 1257314409, 'rmajor': 0, 'machine': ('Linux', 'cmsstor208.fnal.gov', '2.6.31', '#1 SMP Mon Sep 14 09:43:20 CDT 2009', 'x86_64'), 'sanity_size': 65536, 'pstat': (16893, 273565040, 26L, 1, 43789, 5063, 512, 1257314397, 1257314397, 1257282323), 'mode_octal': '0100036', 'mode': 33188, 'gname': 'root', 'size_bytes': 321859275L, 'fullname': '/storage/data1/write-pool-1/data/0010000000000000094E7E80', 'inode': 273579648L, 'minor': 26}, 'version': 'v3_7d CVS $Revision: 1.922 $ encp', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/storage/data1/write-pool-1/data/0010000000000000094E7E80', 'status': ('ok', None), 'outfile': '/pnfs/fnal.gov/usr/cms/WAX/2/nsonmez/pat_v3/Pt0to15/.(access)(0010000000000000094E7E80)', 'fc': {'size': 321859275L, 'external_label': 'VOS175', 'address': ('131.225.13.59', 7501)}, 'infilepath': '/storage/data1/write-pool-1/data/0010000000000000094E7E80', 'at_the_top': 1, 'local_inode': 7148, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.191.79', 49601), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'lm_dequeued': 1257314411.236665, 'encp_start_time': 1257314409.1413119, 'job_queued': 1257314410.0825109, 't0': 1257314409, 'in_queue': 1.1540601253509521}, 'user_level': 'admin', 'unique_id': 'cmsstor208.fnal.gov-1257314409-434-0'} to mover 00:00:11 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM can not remove from sg LTO4_48.mover VOS175 00:00:11 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM SG: {'exp-db': [('LTO4_108.mover', 'VON730'), ('LTO4_41.mover', 'VON240'), ('LTO4_93.mover', 'VON732')], 'cms': [('LTO4_98.mover', 'VOS188'), ('LTO4_51.mover', 'VOQ829'), ('LTO4_45.mover', 'VOS306'), ('LTO4_106.mover', 'VOJ149'), ('LTO4_40.mover', 'VOR657'), ('LTO4_103.mover', 'VOT806'), ('LTO4_44.mover', 'VOR980'), ('LTO4_107.mover', 'VOS312'), ('LTO4_105.mover', 'VOS309'), ('LTO4_46.mover', 'VOR962'), ('LTO4_102.mover', 'VOS315'), ('LTO4_47.mover', 'VOS303'), ('LTO4_96.mover', 'VOS019'), ('LTO4_49.mover', 'VOR958'), ('LTO4_95.mover', 'VOR957'), ('LTO4_109.mover', 'VOQ810')], 'minos': [('LTO4_97.mover', 'VON178'), ('LTO4_110.mover', 'VON129')]} 00:00:11 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM can not remove from vf LTO4_48.mover VOS175 00:00:11 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM VF: {'exp-db.daily-d0-offline.cpio_odc': [('LTO4_108.mover', 'VON730'), ('LTO4_93.mover', 'VON732')], 'cms.NonCustodial.cpio_odc': [('LTO4_51.mover', 'VOQ829'), ('LTO4_109.mover', 'VOQ810')], 'minos.reco_near_dogwood0_sntp.cpio_odc': [('LTO4_97.mover', 'VON178')], 'cms.NonCustodial.cern': [('LTO4_103.mover', 'VOT806')], 'cms.cms2.cpio_odc': [('LTO4_98.mover', 'VOS188'), ('LTO4_47.mover', 'VOS303')], 'cms.cms11.cern': [('LTO4_106.mover', 'VOJ149')], 'cms.cms11.cpio_odc': [('LTO4_45.mover', 'VOS306'), ('LTO4_40.mover', 'VOR657'), ('LTO4_44.mover', 'VOR980'), ('LTO4_107.mover', 'VOS312'), ('LTO4_105.mover', 'VOS309'), ('LTO4_46.mover', 'VOR962'), ('LTO4_102.mover', 'VOS315'), ('LTO4_96.mover', 'VOS019'), ('LTO4_49.mover', 'VOR958'), ('LTO4_95.mover', 'VOR957')], 'exp-db.archive-cdf-offline.cpio_odc': [('LTO4_41.mover', 'VON240')], 'minos.reco_near_dogwood0_mrnt.cpio_odc': [('LTO4_110.mover', 'VON129')]} 00:00:11 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:11 stkenmvr148a.fnal.gov 006010 root I LTO4_48MV WRITE_TO_HSM Thread MainThread 00:00:11 stkenmvr148a.fnal.gov 006010 root I LTO4_48MV SENDING {'vc': {'storage_group': 'cms', 'library': 'CD-LTO4G1', 'file_family': 'cms2', 'wrapper': 'cpio_odc', 'address': ('131.225.13.59', 7502), 'volume_family': 'cms.cms2.cpio_odc', 'file_family_width': 7}, 'outfilepath': '/pnfs/fnal.gov/usr/cms/WAX/2/nsonmez/pat_v3/Pt0to15/QCDDiJet_Pt0to15_patLayer1_fromAOD_full_v2_288_C382344.root', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 25, 'delayed_dismount': None, 'curpri': 25}, 'file_size': 321859275L, 'mover': {'max_buffer': 2621440000L, 'check_written_file': 600, 'max_failures': 5, 'library': 'CD-LTO4G1.library_manager', 'local_mover': 0, 'dismount_delay': 30, 'max_consecutive_failures': 3, 'norestart': 'INQ', 'port': 7848, 'mc_device': '0,5,1,15', 'compression': 0, 'serial_num': '9310036898', 'update_interval_in_bound': 2, 'update_interval': 5, 'syslog_entry': 'sense|st[0-9]', 'status': ('ok', None), 'max_rate': 73400320.0, 'vendor_id': 'IBM', 'driver': 'FTTDriver', 'host': 'stkenmvr148a', 'max_time_in_state': 1200, 'device': 'stkenmvr148a:/dev/rmt/tps0d0n', 'name': 'LTO4_48.mover', 'send_stats': 1, 'product_id': 'ULTRIUM-TD4', 'statistics_path': '/tmp/enstore/enstore/LTO4_48.stat', 'single_filemark': 1, 'mount_delay': 15, 'mover_address': ('131.225.13.176', 7848), 'check_first_written_file': 0, 'callback_addr': ('131.225.13.176', 39299), 'data_ip': 'stkenmvr148a', 'logname': 'LTO4_48MV', 'hostip': '131.225.13.176', 'do_cleaning': 'No', 'media_changer': 'SL8500G1.media_changer', 'min_buffer': 67108864L}, 'ignore_fair_share': None, 'retry': None, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fnal.gov/usr/cms/WAX/2/nsonmez/pat_v3/Pt0to15/QCDDiJet_Pt0to15_patLayer1_fromAOD_full_v2_288_C382344.root', 'uid': 0, 'uname': 'root', 'mtime': 1257314409, 'rmajor': 0, 'machine': ('Linux', 'cmsstor208.fnal.gov', '2.6.31', '#1 SMP Mon Sep 14 09:43:20 CDT 2009', 'x86_64'), 'sanity_size': 65536, 'type': 'cpio_odc', 'pstat': (16893, 273565040, 26L, 1, 43789, 5063, 512, 1257314397, 1257314397, 1257282323), 'mode_octal': '0100036', 'mode': 33188, 'size_bytes': 321859275L, 'gname': 'root', 'gid': 0, 'fullname': '/storage/data1/write-pool-1/data/0010000000000000094E7E80', 'inode': 273579648L, 'minor': 26}, 'version': 'v3_7d CVS $Revision: 1.922 $ encp', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/storage/data1/write-pool-1/data/0010000000000000094E7E80', 'status': ('ok', None), 'outfile': '/pnfs/fnal.gov/usr/cms/WAX/2/nsonmez/pat_v3/Pt0to15/.(access)(0010000000000000094E7E80)', 'fc': {'size': 321859275L, 'external_label': 'VOS175', 'address': ('131.225.13.59', 7501)}, 'infilepath': '/storage/data1/write-pool-1/data/0010000000000000094E7E80', 'at_the_top': 1, 'local_inode': 7148, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.191.79', 49601), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'lm_dequeued': 1257314411.236665, 'encp_start_time': 1257314409.1413119, 'job_queued': 1257314410.0825109, 't0': 1257314409, 'in_queue': 1.1540601253509521}, 'user_level': 'admin', 'unique_id': 'cmsstor208.fnal.gov-1257314409-434-0'} Thread client_connect_thread 00:00:11 cmsstor208.fnal.gov 000434 root I ENCP Control socket ('131.225.191.79', 49601) is connected to ('131.225.13.176', 39300) for cmsstor208.fnal.gov-1257314409-434-0. elapsed=2.140sec 00:00:11 cmsstor208.fnal.gov 000434 root I ENCP Received callback ticket from mover LTO4_48.mover for transfer cmsstor208.fnal.gov-1257314409-434-0. elapsed=2.140sec 00:00:11 cmsstor208.fnal.gov 000434 root I ENCP Atempting to connect data socket to mover at ('131.225.13.176', 39299). 00:00:11 cmsstor208.fnal.gov 000434 root I ENCP Connecting data socket to mover (('131.225.13.176', 39299)) with interface 131.225.191.79. elapsed=2.140sec 00:00:11 cmsstor208.fnal.gov 000434 root I ENCP Data socket ('131.225.191.79', 55965) is connected to ('131.225.13.176', 39299). elapsed=2.140sec 00:00:11 cmsstor208.fnal.gov 000434 root I ENCP Opening local file. 00:00:11 cmsstor208.fnal.gov 000434 root I ENCP pf_status = ('ok', None) skip_layer_cleanup = False 00:00:11 stkenmvr148a.fnal.gov 006010 root I LTO4_48MV MSG_TYPE=MC_LOAD_REQ mounting VOS175 ULTRIUM-TD4 00:00:11 Thread media_thread 00:00:11 stkensrv4n.fnal.gov 003689 enstore I SL8500G1MC PROCESSING mount VOS175 0,5,1,15 00:00:11 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM 00:00:11 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:12 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM 00:00:12 cdfenmvr192a.fnal.gov 006151 root I LTO4_92MV written bytes 251392/251392, blocks 2 header 0 trailer 52 Thread tape_thread 00:00:12 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:12 stkensrv4n.fnal.gov 005852 enstore I STKMC FINISHED listDrives returned ('ok', 0, None, '', '') 00:00:13 stkensrv4n.fnal.gov 004406 enstore I SL8500G1MC query drive 0,5,1,15 => 0,0,5,1,15 online available IBM-LTO4 00:00:13 stkensrv4n.fnal.gov 004406 enstore I SL8500G1MC Dismount 14 ignored: Drive 0,5,1,15 is empty. Thought VOS175 was there =>. dismount VOLUME 0,5,1,15 force => ok,0,5,1,15 online available IBM-LTO4 00:00:13 cmsstor62.fnal.gov 023696 root I ENCP Version: v3_7d CVS $Revision: 1.922 $ encp OS: Linux 2.6.31 x86_64 Release: Scientific Linux SL release 5.2 (Boron) Rocks release 5.1 (V.I) User: root(0) Group: root(0) Euser: root(0) Egroup: root(0) Library: CD-LTO4G1 Storage Group: cms File Family: cms11 FF Wrapper: cpio_odc FF Width: 10 Current working directory: cmsstor62.fnal.gov:/ Command line: /home/enstore/enstore/bin/encp --verbose 4 --threaded --ecrc --resubmit-timeout 1800 --priority 25 --pnfs-mount /pnfs/fs --put-cache 000700000000000062568AE8 /storage/data2/write-pool-2/data/000700000000000062568AE8 00:00:14 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM 00:00:14 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:14 cmsstor62.fnal.gov 023696 root I ENCP Creating zero length output files. 00:00:14 cmsstor62.fnal.gov 023696 root I ENCP Sending /pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0087/08FCCDCE-F1C8-DE11-BDAD-002618943884.root write request to LM: unique_id: cmsstor62.fnal.gov-1257314414-23696-0 inputfile: /storage/data2/write-pool-2/data/000700000000000062568AE8 outputfile: /pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0087/.(access)(000700000000000062568AE8) 00:00:14 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM MSG_TYPE=ADD_TO_LMQ request added to pending queue for storage group : cms 00:00:14 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM write Q'd /storage/data2/write-pool-2/data/000700000000000062568AE8 (cmsstor62.fnal.gov-1257314414-23696-0) -> /pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0087/08FCCDCE-F1C8-DE11-BDAD-002618943884.root : library=CD-LTO4G1 family=cms11 requester:root volume_family:cms.cms11.cpio_odc 00:00:14 cmsstor62.fnal.gov 023696 root I ENCP Listening for control socket at: ('131.225.188.4', 46510) 00:00:14 cmsstor62.fnal.gov 023696 root I ENCP Waiting for mover to connect control socket at address: ('131.225.188.4', 46510) elapsed=1.960sec 00:00:14 stkendca22a.fnal.gov 014473 root I ENCP Version: v3_8b CVS $Revision: 1.922 $ OS: Linux 2.6.9-89.0.9.ELsmp i686 Release: Scientific Linux Fermi LTS release 4.4 (Wilson) User: root(0) Group: root(0) Euser: root(0) Egroup: root(0) Library: garbage Storage Group: test File Family: dcache FF Wrapper: cpio_odc FF Width: 1 Current working directory: stkendca22a.fnal.gov:/home/enstore/dcache-log Command line: /fnal/ups/prd/encp/v3_8b/Linux-2-6-dcache/encp --verbose 4 --threaded --ecrc --bypass-filesystem-max-filesize-check --pnfs-mount /pnfs/fs --shortcut --override-path /pnfs/fnal.gov/usr/fermigrid/volatile/fermilab/timur-test/broken-lib/timur-test-0003 --put-cache 0005000000000000005FA5F8 /diskb/write-pool-2/data/0005000000000000005FA5F8 00:00:14 stkendca22a.fnal.gov 014533 root I ENCP Version: v3_8b CVS $Revision: 1.922 $ OS: Linux 2.6.9-89.0.9.ELsmp i686 Release: Scientific Linux Fermi LTS release 4.4 (Wilson) User: root(0) Group: root(0) Euser: root(0) Egroup: root(0) Library: garbage Storage Group: test File Family: dcache FF Wrapper: cpio_odc FF Width: 1 Current working directory: stkendca22a.fnal.gov:/home/enstore/dcache-log Command line: /fnal/ups/prd/encp/v3_8b/Linux-2-6-dcache/encp --verbose 4 --threaded --ecrc --bypass-filesystem-max-filesize-check --pnfs-mount /pnfs/fs --shortcut --override-path /pnfs/fnal.gov/usr/fermigrid/volatile/fermilab/timur-test/broken-lib/timur-test-0001 --put-cache 0005000000000000005FA590 /diskb/write-pool-2/data/0005000000000000005FA590 00:00:14 cmsstor81.fnal.gov 015087 root I ENCP File /storage/data3/write-pool-3/data/000700000000000062342618 transfered for cmsstor81.fnal.gov-1257312779-15087-0 in 20.726099968 seconds. 00:00:15 stkendca22a.fnal.gov 014561 root I ENCP Version: v3_8b CVS $Revision: 1.922 $ OS: Linux 2.6.9-89.0.9.ELsmp i686 Release: Scientific Linux Fermi LTS release 4.4 (Wilson) User: root(0) Group: root(0) Euser: root(0) Egroup: root(0) Library: garbage Storage Group: test File Family: dcache FF Wrapper: cpio_odc FF Width: 1 Current working directory: stkendca22a.fnal.gov:/home/enstore/dcache-log Command line: /fnal/ups/prd/encp/v3_8b/Linux-2-6-dcache/encp --verbose 4 --threaded --ecrc --bypass-filesystem-max-filesize-check --pnfs-mount /pnfs/fs --shortcut --override-path /pnfs/fnal.gov/usr/fermigrid/volatile/fermilab/timur-test/broken-lib/timur-test-0002 --put-cache 0005000000000000005FA5C8 /diskb/write-pool-2/data/0005000000000000005FA5C8 00:00:15 stkensrv4n.fnal.gov 005357 enstore E LTO4G1LM MOVER ERROR RQ {'status': ('ok', None), 'returned_work': None, 'library': None, 'mover_type': '', 'current_location': '0000_000000000_0000000', 'ip_map': '', 'operation': None, 'read_only': 0, 'current_time': 1257314415.3302529, 'error_source': None, 'external_label': None, 'state': 'OFFLINE', 'r_a': (('131.225.13.194', 54880), 14550L, '131.225.13.194-54880-1256586929.509013-4866'), 'library_list': [('CD-LTO4G1.library_manager', ('131.225.13.4', 8541))], 'current_priority': (None, None), 'transfer_deficiency': 1, 'time_in_state': 727484.97019481659, 'address': ('131.225.13.194', 7860), 'volume_family': None, 'work': 'mover_error', 'mover': 'LTO4_99.mover', 'volume_status': (['none', 'none'], ['none', 'none']), 'unique_id': None} 00:00:15 stkendca22a.fnal.gov 014533 root I ENCP Configuration Server: no such name: 'garbage.library_manager' 00:00:15 stkendca22a.fnal.gov 014533 root E ENCP INFILE=/diskb/write-pool-2/data/0005000000000000005FA590 OUTFILE=/pnfs/fnal.gov/usr/fermigrid/volatile/fermilab/timur-test/broken-lib/timur-test-0001 FILESIZE=801512 LABEL= LOCATION= DRIVE= DRIVE_SN= TRANSFER_TIME=0.00 SEEK_TIME=0.00 MOUNT_TIME=0.00 QWAIT_TIME=0.00 TIME2NOW=1.84 CRC= STATUS=KEYERROR Configuration Server: no such name: 'garbage.library_manager' 00:00:15 stkendca22a.fnal.gov 014473 root I ENCP Configuration Server: no such name: 'garbage.library_manager' 00:00:15 stkendca22a.fnal.gov 014473 root E ENCP INFILE=/diskb/write-pool-2/data/0005000000000000005FA5F8 OUTFILE=/pnfs/fnal.gov/usr/fermigrid/volatile/fermilab/timur-test/broken-lib/timur-test-0003 FILESIZE=801512 LABEL= LOCATION= DRIVE= DRIVE_SN= TRANSFER_TIME=0.00 SEEK_TIME=0.00 MOUNT_TIME=0.00 QWAIT_TIME=0.00 TIME2NOW=1.91 CRC= STATUS=KEYERROR Configuration Server: no such name: 'garbage.library_manager' 00:00:16 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM 00:00:16 stkensrv4n.fnal.gov 004406 enstore I SL8500G1MC query vol VOS175 => 0,VOS175 home 0, 6,10,20, 1 LTO-800G 00:00:16 stkendca22a.fnal.gov 014561 root I ENCP Configuration Server: no such name: 'garbage.library_manager' 00:00:16 stkendca22a.fnal.gov 014561 root E ENCP INFILE=/diskb/write-pool-2/data/0005000000000000005FA5C8 OUTFILE=/pnfs/fnal.gov/usr/fermigrid/volatile/fermilab/timur-test/broken-lib/timur-test-0002 FILESIZE=801512 LABEL= LOCATION= DRIVE= DRIVE_SN= TRANSFER_TIME=0.00 SEEK_TIME=0.00 MOUNT_TIME=0.00 QWAIT_TIME=0.00 TIME2NOW=2.07 CRC= STATUS=KEYERROR Configuration Server: no such name: 'garbage.library_manager' 00:00:16 cmsstor153.fnal.gov 014680 root I ENCP File /storage/data2/write-pool-2/data/000700000000000062552850 transfered for cmsstor153.fnal.gov-1257312788-14680-0 in 20.4999170303 seconds. 00:00:16 stkenmvr196a.fnal.gov 027637 root I LTO4_96MV written bytes 2234260480/2234260480, blocks 17047 header 192 trailer 279 Thread tape_thread 00:00:16 cdfenmvr192a.fnal.gov 006151 root I LTO4_92MV filemarks written. Tape VOV138 absolute location in blocks 255455 Thread tape_thread 00:00:16 cdfenmvr192a.fnal.gov 006151 root I LTO4_92MV write_tape timing:pull 1.81198120117e-05 write 0.00448894500732 crc_check 0.00143003463745 freespace 1.38282775879e-05 block_write 0.00595092773438 idle 3.60012054443e-05 Thread tape_thread 00:00:16 cdfenmvr192a.fnal.gov 006151 root I LTO4_92MV drive stats after write. Tape VOV138 position 5179 block 0 block_size 0 bloc_loc 255455 tot_blocks 762939000 BOT 0 read_err 0 write_err 14 bytes 251392 block_write_tot 0.00595092773438 tape_rate 42244169.5179 Thread tape_thread 00:00:16 cdfenmvr192a.fnal.gov 006151 root I LTO4_92MV remaining bytes info in DB 738928640000 estimated 738928388608 reported from drive 738923520000 position 5182 Thread tape_thread 00:00:16 cdfenmvr192a.fnal.gov 006151 root I LTO4_92MV new bitfile request {'uid': 0, 'external_label': 'VOV138', 'gid': 0, 'location_cookie': '0000_000000000_0005180', 'complete_crc': 4169464483L, 'sanity_cookie': (65536L, 946328236L), 'size': 251340L} Thread tape_thread 00:00:16 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:16 cdfenmvr192a.fnal.gov 006151 root I LTO4_92MV set remaining: VOV138 738923520000 0000_000000000_0005182 Thread tape_thread 00:00:16 last message repeated 1 times 00:00:16 stkensrv0n.fnal.gov 028762 enstore I VOLSRV Updating update volume set last_access = '2009-11-04 00:00:16', eod_cookie = '0000_000000000_0005182', non_del_files = '1728', remaining_bytes = 738923520000 where label = 'VOV138'; 00:00:16 cdfenmvr192a.fnal.gov 006151 root I LTO4_92MV transfer complete volume=VOV138 location=5182 Thread tape_thread 00:00:16 last message repeated 1 times 00:00:16 stkenmvr196a.fnal.gov 027637 root I LTO4_96MV filemarks written. Tape VOS019 absolute location in blocks 1858507 Thread tape_thread 00:00:16 stkenmvr196a.fnal.gov 027637 root I LTO4_96MV write_tape timing:pull 0.190261125565 write 3.56445717812 crc_check 10.4769024849 freespace 0.212735652924 block_write 14.4443564415 idle 0.351608037949 Thread tape_thread 00:00:16 stkenmvr196a.fnal.gov 027637 root I LTO4_96MV drive stats after write. Tape VOS019 position 81 block 0 block_size 0 bloc_loc 1858507 tot_blocks 762939000 BOT 0 read_err 0 write_err 0 bytes 2234260480 block_write_tot 14.4443564415 tape_rate 154680514.085 Thread tape_thread 00:00:16 stkensrv0n.fnal.gov 028762 enstore I VOLSRV Updating update volume set sum_wr_access = '1727', modification_time = '2009-11-04 00:00:16' where label = 'VOV138'; 00:00:16 cdfenmvr192a.fnal.gov 006151 root I LTO4_92MV Sending done to client: {'vc': {'comment': '', 'storage_group': 'ktev', 'blocksize': 131072, 'declared': 1254165875.0, 'library': 'CD-LTO4F1', 'si_time': [0.0, 0.0], 'user_inhibit': ['none', 'none'], 'system_inhibit': ['none', 'none'], 'external_label': 'VOV138', 'wrapper': 'cern', 'remaining_bytes': 738928640000L, 'sum_mounts': 1, 'sum_rd_access': 0, 'file_family_width': 1, 'last_access': 1257314409.0, 'media_type': 'LTO4', 'status': ('ok', None), 'eod_cookie': '0000_000000000_0005179', 'capacity_bytes': 858993459200L, 'non_del_files': 1727, 'sum_wr_err': 0, 'sum_wr_access': 1726, 'file_family': 'ktev_migration-MIGRATION', 'address': ('131.225.13.59', 7502), 'volume_family': 'ktev.ktev_migration-MIGRATION.cern', 'modification_time': 1257314409.0, 'write_protected': 'n', 'sum_rd_err': 0, 'first_access': 1257302588.0}, 'outfilepath': '/pnfs/fs/usr/Migration/ktev/migrated_from_dlt/KQ/KQRC/KQRC38/f6698', 'encp': {'agetime': 30, 'adminpri': 6, 'delpri': 1, 'basepri': 1000007, 'delayed_dismount': 2, 'curpri': 1000007}, 'file_size': 251340L, 'mover': {'max_buffer': 2621440000L, 'check_written_file': 600, 'max_failures': 5, 'library': 'CD-LTO4F1.library_manager', 'local_mover': 0, 'single_filemark': 1, 'max_consecutive_failures': 3, 'norestart': 'INQ', 'port': 7853, 'mc_device': '1,0,1,1', 'compression': 0, 'serial_num': '1310060800', 'update_interval_in_bound': 2, 'update_interval': 5, 'syslog_entry': 'sense|st[0-9]', 'status': ('ok', None), 'max_rate': 73400320.0, 'vendor_id': 'IBM', 'driver': 'FTTDriver', 'host': 'cdfenmvr192a', 'max_time_in_state': 1200, 'device': 'cdfenmvr192a:/dev/rmt/tps0d0n', 'name': 'LTO4_92.mover', 'send_stats': 1, 'product_id': 'ULTRIUM-TD4', 'statistics_path': '/tmp/enstore/enstore/LTO4_92.stat', 'mover_address': ('131.225.215.76', 7853), 'mount_delay': 15, 'check_first_written_file': 0, 'callback_addr': ('131.225.215.76', 60650), 'data_ip': 'cdfenmvr192a', 'logname': 'LTO4_92MV', 'hostip': '131.225.215.76', 'do_cleaning': 'No', 'media_changer': 'SL8500F1.media_changer'}, 'ignore_fair_share': 1, 'retry': None, 'lm': {'address': ('131.225.13.4', 8542)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fs/usr/ktev/migrated_from_dlt/KQ/KQRC/KQRC38/f6698', 'uid': 0, 'uname': 'root', 'mtime': 1257314397, 'rmajor': 0, 'machine': ('Linux', 'stkendm7a.fnal.gov', '2.6.9-89.0.9.ELsmp', '#1 SMP Mon Aug 24 13:47:43 CDT 2009', 'i686'), 'sanity_size': 65536, 'type': 'cern', 'pstat': (16877, 678574944L, 20L, 1, 0, 0, 512L, 1257314008, 1257314008, 1252942702), 'mode_octal': '0100016', 'mode': 33188, 'size_bytes': 251340L, 'gname': 'root', 'gid': 0, 'fullname': '/data/data2/Migration_Spool/VO6153:0000_000000000_0032549', 'inode': 675389728L, 'minor': 20}, 'version': 'v3_8 CVS $Revision: 1.963 $ encp.pyc', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/data/data2/Migration_Spool/VO6153:0000_000000000_0032549', 'status': ('ok', None), 'outfile': '/pnfs/fs/usr/Migration/ktev/migrated_from_dlt/KQ/KQRC/KQRC38/.(access)(00280000000000000841A120)', 'fc': {'uid': 0, 'external_label': 'VOV138', 'gid': 0, 'location_cookie': '0000_000000000_0005180', 'complete_crc': 4169464483L, 'bfid': 'CDMS125731441600000', 'sanity_cookie': (65536L, 946328236L), 'size': 251340L}, 'infilepath': '/data/data2/Migration_Spool/VO6153:0000_000000000_0032549', 'at_the_top': 45, 'local_inode': 4495418L, 'resend': {'max_retry': 3, 'resubmits': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.13.19', 57439), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'encp_start_time': 1257314397.8037109, 'seek_time': 0.026788949966430664, 'transfer_time': 6.1454601287841797, 't0': 1257314397, 'mount_time': 0.018427133560180664, 'lm_dequeued': 1257314410.4020441, 'job_queued': 1257314398.007931, 'drive_call_time': 0.015519857406616211, 'in_queue': 12.393855094909668, 'drive_transfer_time': 5.8888351917266846}, 'user_level': 'admin', 'unique_id': 'stkendm7a.fnal.gov-1257314397-18643-24631'} Thread tape_thread 00:00:16 cdfenmvr192a.fnal.gov 006151 root I LTO4_92MV Done is sent Thread tape_thread 00:00:16 cdfenmvr192a.fnal.gov 006151 root I LTO4_92MV maybe_clean Thread tape_thread 00:00:16 stkendm7a.fnal.gov 018643 root I ENCP Received final dialog for stkendm7a.fnal.gov-1257314397-18643-24631. elapsed=154663.110sec Thread WRITE_1 00:00:16 stkendm7a.fnal.gov 018643 root I ENCP leaving transfer_file(). Thread WRITE_1 00:00:16 stkenmvr196a.fnal.gov 027637 root I LTO4_96MV remaining bytes info in DB 552622080000 estimated 550387819520 reported from drive 550541312000 position 82 Thread tape_thread 00:00:16 stkenmvr196a.fnal.gov 027637 root I LTO4_96MV new bitfile request {'uid': 0, 'external_label': 'VOS019', 'gid': 0, 'location_cookie': '0000_000000000_0000081', 'complete_crc': 32546453L, 'sanity_cookie': (65536L, 3625856147L), 'size': 2234260009L} Thread tape_thread 00:00:16 cdfenmvr192a.fnal.gov 006151 root I LTO4_92MV returned from maybe_clean Thread tape_thread 00:00:16 stkenmvr196a.fnal.gov 027637 root I LTO4_96MV set remaining: VOS019 550541312000 0000_000000000_0000082 Thread tape_thread 00:00:16 last message repeated 1 times 00:00:16 stkensrv0n.fnal.gov 028762 enstore I VOLSRV Updating update volume set last_access = '2009-11-04 00:00:16', eod_cookie = '0000_000000000_0000082', non_del_files = '81', remaining_bytes = 550541312000 where label = 'VOS019'; 00:00:16 stkenmvr196a.fnal.gov 027637 root I LTO4_96MV transfer complete volume=VOS019 location=82 Thread tape_thread 00:00:16 last message repeated 1 times 00:00:16 stkensrv0n.fnal.gov 028762 enstore I VOLSRV Updating update volume set sum_wr_access = '81', modification_time = '2009-11-04 00:00:16' where label = 'VOS019'; 00:00:16 stkensrv0n.fnal.gov 028796 enstore I FILSRV Updating update file set deleted = 'n', pnfs_path = '/pnfs/fs/usr/ktev/migrated_from_dlt/KQ/KQRC/KQRC38/f6698', drive = 'cdfenmvr192a:/dev/rmt/tps0d0n:1310060800', pnfs_id = '00280000000000000841A120' where bfid = 'CDMS125731441600000'; 00:00:16 stkenmvr196a.fnal.gov 027637 root I LTO4_96MV Sending done to client: {'vc': {'comment': '', 'storage_group': 'cms', 'blocksize': 131072, 'declared': 1255359355.0, 'library': 'CD-LTO4G1', 'si_time': [0.0, 1252643467.0], 'user_inhibit': ['none', 'none'], 'system_inhibit': ['none', 'none'], 'external_label': 'VOS019', 'wrapper': 'cpio_odc', 'remaining_bytes': 552622080000L, 'sum_mounts': 19, 'sum_rd_access': 0, 'file_family_width': 10, 'last_access': 1257314394.0, 'media_type': 'LTO4', 'status': ('ok', None), 'eod_cookie': '0000_000000000_0000081', 'capacity_bytes': 858993459200L, 'non_del_files': 80, 'sum_wr_err': 0, 'sum_wr_access': 80, 'file_family': 'cms11', 'address': ('131.225.13.59', 7502), 'volume_family': 'cms.cms11.cpio_odc', 'modification_time': 1257314394.0, 'write_protected': 'n', 'sum_rd_err': 0, 'first_access': 1257311250.0}, 'outfilepath': '/pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0087/0C4C9FEB-F0C8-DE11-89D1-003048678B70.root', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 25, 'delayed_dismount': None, 'curpri': 25}, 'file_size': 2234260009L, 'mover': {'max_buffer': 2621440000L, 'check_written_file': 600, 'max_failures': 5, 'library': 'CD-LTO4G1.library_manager', 'local_mover': 0, 'single_filemark': 1, 'max_consecutive_failures': 3, 'norestart': 'INQ', 'port': 7857, 'mc_device': '0,7,1,2', 'compression': 0, 'serial_num': '9310108548', 'update_interval_in_bound': 2, 'update_interval': 5, 'syslog_entry': 'sense|st[0-9]', 'status': ('ok', None), 'max_rate': 73400320.0, 'vendor_id': 'IBM', 'driver': 'FTTDriver', 'host': 'stkenmvr196a', 'max_time_in_state': 1200, 'device': 'stkenmvr196a:/dev/rmt/tps0d0n', 'name': 'LTO4_96.mover', 'send_stats': 1, 'product_id': 'ULTRIUM-TD4', 'statistics_path': '/tmp/enstore/enstore/LTO4_96.stat', 'mover_address': ('131.225.13.191', 7857), 'mount_delay': 15, 'check_first_written_file': 0, 'callback_addr': ('131.225.13.191', 38297), 'data_ip': 'stkenmvr196a', 'logname': 'LTO4_96MV', 'hostip': '131.225.13.191', 'do_cleaning': 'No', 'media_changer': 'SL8500G1.media_changer'}, 'ignore_fair_share': None, 'retry': None, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0087/0C4C9FEB-F0C8-DE11-89D1-003048678B70.root', 'uid': 0, 'uname': 'root', 'mtime': 1257312788, 'rmajor': 0, 'machine': ('Linux', 'cmsstor153.fnal.gov', '2.6.31', '#1 SMP Mon Sep 14 09:43:20 CDT 2009', 'x86_64'), 'sanity_size': 65536, 'type': 'cpio_odc', 'pstat': (16893, 122945816, 21L, 1, 9811, 5063, 512, 1257307721, 1257307721, 1257304542), 'mode_octal': '0100036', 'mode': 33188, 'size_bytes': 2234260009L, 'gname': 'root', 'gid': 0, 'fullname': '/storage/data2/write-pool-2/data/000700000000000062552850', 'inode': 123021392L, 'minor': 21}, 'version': 'v3_7d CVS $Revision: 1.922 $ encp', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/storage/data2/write-pool-2/data/000700000000000062552850', 'status': ('ok', None), 'outfile': '/pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0087/.(access)(000700000000000062552850)', 'fc': {'uid': 0, 'external_label': 'VOS019', 'gid': 0, 'location_cookie': '0000_000000000_0000081', 'complete_crc': 32546453L, 'bfid': 'CDMS125731441600001', 'sanity_cookie': (65536L, 3625856147L), 'size': 2234260009L}, 'infilepath': '/storage/data2/write-pool-2/data/000700000000000062552850', 'at_the_top': 6, 'local_inode': 2074, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.189.124', 50940), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'encp_start_time': 1257312787.416939, 'seek_time': 0.031509160995483398, 'transfer_time': 21.13127589225769, 't0': 1257312787, 'mount_time': 0.013940811157226562, 'lm_dequeued': 1257314395.5783739, 'job_queued': 1257312788.296983, 'drive_call_time': 3.3141593933105469, 'in_queue': 1607.2810730934143, 'drive_transfer_time': 15.459670066833496}, 'user_level': 'admin', 'unique_id': 'cmsstor153.fnal.gov-1257312788-14680-0'} Thread tape_thread 00:00:16 stkenmvr196a.fnal.gov 027637 root I LTO4_96MV Done is sent Thread tape_thread 00:00:16 stkenmvr196a.fnal.gov 027637 root I LTO4_96MV maybe_clean Thread tape_thread 00:00:16 cmsstor153.fnal.gov 014680 root I ENCP Received final dialog for cmsstor153.fnal.gov-1257312788-14680-0. elapsed=1629.290sec 00:00:16 cmsstor153.fnal.gov 014680 root I ENCP leaving transfer_file(). 00:00:16 stkenmvr196a.fnal.gov 027637 root I LTO4_96MV returned from maybe_clean Thread tape_thread 00:00:16 cmsstor153.fnal.gov 014680 root I ENCP pf_status = ('ok', None) skip_layer_cleanup = False 00:00:17 stkensrv0n.fnal.gov 028796 enstore I FILSRV Updating update file set deleted = 'n', pnfs_path = '/pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0087/0C4C9FEB-F0C8-DE11-89D1-003048678B70.root', drive = 'stkenmvr196a:/dev/rmt/tps0d0n:9310108548', pnfs_id = '000700000000000062552850' where bfid = 'CDMS125731441600001'; 00:00:17 stkendm7a.fnal.gov 018643 root I ENCP MSG_TYPE=ENCP_XFER write_to_hsm /data/data2/Migration_Spool/VO6153:0000_000000000_0032549 -> /pnfs/fs/usr/Migration/ktev/migrated_from_dlt/KQ/KQRC/KQRC38/f6698: 251340 bytes copied to VOV138 at 0.0388 MB/S (80.7 MB/S network) (0.0407 MB/S drive) (104 MB/S disk) mover=LTO4_92.mover drive_id=ULTRIUM-TD4 drive_sn=1310060800 drive_vendor=IBM elapsed=19.335 {'storage_group': 'ktev', 'encp_crc': 4169464483L, 'disk_rate': 103.9898071692546, 'drive_rate': 0.040703550852008934, 'driver': 'FTTDriver', 'transfer_rate': 0.03877508886439969, 'mover_interface': 'cdfenmvr192a', 'overall_rate': 0.035759346173348742, 'media_changer': 'SL8500F1.media_changer', 'network_rate': 0.03877508886439969, 'encp_ip': '131.225.13.19', 'unique_id': 'stkendm7a.fnal.gov-1257314397-18643-24631'} Thread WRITE_1 00:00:17 stkendm7a.fnal.gov 018643 root I ENCP File /pnfs/fs/usr/Migration/ktev/migrated_from_dlt/KQ/KQRC/KQRC38/f6698 copied successfully. Thread WRITE_1 00:00:17 stkendm7a.fnal.gov 018643 root I ENCP Completed transferring 251340 bytes in 1 files in 19.3363511562 sec. Overall rate = 0.0358 MB/sec. Transfer rate = 0.0388 MB/sec. Network rate = 80.7 MB/sec. Drive rate = 0.0407 MB/sec. Disk rate = 104 MB/sec. Exit status = 0. Thread WRITE_1 00:00:17 cmsstor153.fnal.gov 014680 root I ENCP MSG_TYPE=ENCP_XFER write_to_hsm /storage/data2/write-pool-2/data/000700000000000062552850 -> /pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0087/0C4C9FEB-F0C8-DE11-89D1-003048678B70.root: 2234260009 bytes copied to VOS019 at 101 MB/S (92.9 MB/S network) (138 MB/S drive) (104 MB/S disk) mover=LTO4_96.mover drive_id=ULTRIUM-TD4 drive_sn=9310108548 drive_vendor=IBM elapsed=1629.9 {'storage_group': 'cms', 'encp_crc': 32546453L, 'disk_rate': 103.96630110951219, 'drive_rate': 137.82676975287649, 'driver': 'FTTDriver', 'transfer_rate': 100.75329504469737, 'mover_interface': 'stkenmvr196a', 'overall_rate': 98.206396129641405, 'media_changer': 'SL8500G1.media_changer', 'network_rate': 100.75329504469737, 'encp_ip': '131.225.189.124', 'unique_id': 'cmsstor153.fnal.gov-1257312788-14680-0'} 00:00:17 cmsstor153.fnal.gov 014680 root I ENCP File /pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0087/0C4C9FEB-F0C8-DE11-89D1-003048678B70.root copied successfully. 00:00:17 cmsstor153.fnal.gov 014680 root I ENCP Completed transferring 2234260009 bytes in 1 files in 1629.89243507 sec. Overall rate = 98.2 MB/sec. Transfer rate = 101 MB/sec. Network rate = 92.9 MB/sec. Drive rate = 138 MB/sec. Disk rate = 104 MB/sec. Exit status = 0. 00:00:17 last message repeated 1 times 00:00:17 stkensrv0n.fnal.gov 028796 enstore I FILSRV start modifying {'pnfsid': '00280000000000000841A120', 'pnfs_name0': '/pnfs/fs/usr/ktev/migrated_from_dlt/KQ/KQRC/KQRC38/f6698', 'deleted': 'no', 'external_label': 'VOV138', 'drive': 'cdfenmvr192a:/dev/rmt/tps0d0n:1310060800', 'update': '2009-11-04 00:00:16.800691', 'gid': 0, 'location_cookie': '0000_000000000_0005180', 'complete_crc': 4169464483L, 'uid': 0, 'bfid': 'CDMS125731441600000', 'sanity_cookie': (65536L, 946328236L), 'size': 251340L} 00:00:17 stkensrv0n.fnal.gov 028796 enstore I FILSRV Updating update file set pnfs_id = '001F00000000000000E2EF30' where bfid = 'CDMS125731441600000'; 00:00:17 stkensrv0n.fnal.gov 028796 enstore I FILSRV modified to {'pnfsid': '001F00000000000000E2EF30', 'pnfs_name0': '/pnfs/fs/usr/ktev/migrated_from_dlt/KQ/KQRC/KQRC38/f6698', 'deleted': 'no', 'external_label': 'VOV138', 'drive': 'cdfenmvr192a:/dev/rmt/tps0d0n:1310060800', 'update': '2009-11-04 00:00:16.800691', 'gid': 0, 'location_cookie': '0000_000000000_0005180', 'complete_crc': 4169464483L, 'uid': 0, 'bfid': 'CDMS125731441600000', 'sanity_cookie': (65536L, 946328236L), 'size': 251340L} 00:00:17 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM 00:00:17 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM write_to_hsm next work on vol=VOV138 mover=LTO4_92.mover requester:root 00:00:17 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM HAVE_BOUND:sending write_to_hsm /pnfs/fs/usr/ktev/migrated_from_dlt/KQ/KQRC/KQRC52/f5012 to mover LTO4_92.mover ('131.225.215.76', 7853) DEL_DISM 2 00:00:17 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM MSG_TYPE=ADD_TO_LMQ request added to work at movers queue for storage group : ktev 00:00:17 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM HAVE_BOUND:sending {'vc': {'storage_group': 'ktev', 'external_label': 'VOV138', 'library': 'CD-LTO4F1', 'file_family': 'ktev_migration-MIGRATION', 'wrapper': 'cern', 'address': ('131.225.13.59', 7502), 'volume_family': 'ktev.ktev_migration-MIGRATION.cern', 'file_family_width': 1}, 'outfilepath': '/pnfs/fs/usr/Migration/ktev/migrated_from_dlt/KQ/KQRC/KQRC52/f5012', 'encp': {'agetime': 30, 'adminpri': 6, 'delpri': 1, 'basepri': 1000007, 'delayed_dismount': 2, 'curpri': 1000007}, 'file_size': 477904L, 'mover': 'LTO4_92.mover', 'ignore_fair_share': 1, 'retry': None, 'lm': {'address': ('131.225.13.4', 8542)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fs/usr/ktev/migrated_from_dlt/KQ/KQRC/KQRC52/f5012', 'uid': 0, 'uname': 'root', 'type': 'cern', 'gid': 0, 'mtime': 1257314404, 'rmajor': 0, 'machine': ('Linux', 'stkendm7a.fnal.gov', '2.6.9-89.0.9.ELsmp', '#1 SMP Mon Aug 24 13:47:43 CDT 2009', 'i686'), 'sanity_size': 65536, 'pstat': (16877, 678574248L, 20L, 1, 0, 0, 512L, 1257314403, 1257314403, 1252942658), 'mode_octal': '0100016', 'mode': 33188, 'gname': 'root', 'size_bytes': 477904L, 'fullname': '/data/data2/Migration_Spool/VO6153:0000_000000000_0032300', 'inode': 675389768L, 'minor': 20}, 'version': 'v3_8 CVS $Revision: 1.963 $ encp.pyc', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/data/data2/Migration_Spool/VO6153:0000_000000000_0032300', 'status': ('ok', None), 'outfile': '/pnfs/fs/usr/Migration/ktev/migrated_from_dlt/KQ/KQRC/KQRC52/.(access)(00280000000000000841A148)', 'fc': {'size': 477904L, 'external_label': 'VOV138', 'pnfsid': '00280000000000000841A148', 'address': ('131.225.13.59', 7501)}, 'infilepath': '/data/data2/Migration_Spool/VO6153:0000_000000000_0032300', 'at_the_top': 42, 'local_inode': 4495347L, 'resend': {'max_retry': 3, 'resubmits': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.13.19', 57442), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'lm_dequeued': 1257314417.392499, 'encp_start_time': 1257314403.944859, 'job_queued': 1257314404.256093, 't0': 1257314403, 'in_queue': 13.136147975921631}, 'user_level': 'admin', 'unique_id': 'stkendm7a.fnal.gov-1257314404-18643-24632'} to mover 00:00:17 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM 00:00:17 cdfenmvr192a.fnal.gov 006151 root I LTO4_92MV WRITE_TO_HSM Thread MainThread 00:00:17 stkendm7a.fnal.gov 018643 root I ENCP Listening for control socket at: ('131.225.13.19', 57442) Thread WRITE_2 00:00:17 cdfenmvr192a.fnal.gov 006151 root I LTO4_92MV SENDING {'vc': {'storage_group': 'ktev', 'external_label': 'VOV138', 'library': 'CD-LTO4F1', 'file_family': 'ktev_migration-MIGRATION', 'wrapper': 'cern', 'address': ('131.225.13.59', 7502), 'volume_family': 'ktev.ktev_migration-MIGRATION.cern', 'file_family_width': 1}, 'outfilepath': '/pnfs/fs/usr/Migration/ktev/migrated_from_dlt/KQ/KQRC/KQRC52/f5012', 'encp': {'agetime': 30, 'adminpri': 6, 'delpri': 1, 'basepri': 1000007, 'delayed_dismount': 2, 'curpri': 1000007}, 'file_size': 477904L, 'mover': {'max_buffer': 2621440000L, 'check_written_file': 600, 'max_failures': 5, 'library': 'CD-LTO4F1.library_manager', 'local_mover': 0, 'single_filemark': 1, 'max_consecutive_failures': 3, 'norestart': 'INQ', 'port': 7853, 'mc_device': '1,0,1,1', 'compression': 0, 'serial_num': '1310060800', 'update_interval_in_bound': 2, 'update_interval': 5, 'syslog_entry': 'sense|st[0-9]', 'status': ('ok', None), 'max_rate': 73400320.0, 'vendor_id': 'IBM', 'driver': 'FTTDriver', 'host': 'cdfenmvr192a', 'max_time_in_state': 1200, 'device': 'cdfenmvr192a:/dev/rmt/tps0d0n', 'name': 'LTO4_92.mover', 'send_stats': 1, 'product_id': 'ULTRIUM-TD4', 'statistics_path': '/tmp/enstore/enstore/LTO4_92.stat', 'mover_address': ('131.225.215.76', 7853), 'mount_delay': 15, 'check_first_written_file': 0, 'callback_addr': ('131.225.215.76', 60656), 'data_ip': 'cdfenmvr192a', 'logname': 'LTO4_92MV', 'hostip': '131.225.215.76', 'do_cleaning': 'No', 'media_changer': 'SL8500F1.media_changer'}, 'ignore_fair_share': 1, 'retry': None, 'lm': {'address': ('131.225.13.4', 8542)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fs/usr/ktev/migrated_from_dlt/KQ/KQRC/KQRC52/f5012', 'uid': 0, 'uname': 'root', 'mtime': 1257314404, 'rmajor': 0, 'machine': ('Linux', 'stkendm7a.fnal.gov', '2.6.9-89.0.9.ELsmp', '#1 SMP Mon Aug 24 13:47:43 CDT 2009', 'i686'), 'sanity_size': 65536, 'type': 'cern', 'pstat': (16877, 678574248L, 20L, 1, 0, 0, 512L, 1257314403, 1257314403, 1252942658), 'mode_octal': '0100016', 'mode': 33188, 'size_bytes': 477904L, 'gname': 'root', 'gid': 0, 'fullname': '/data/data2/Migration_Spool/VO6153:0000_000000000_0032300', 'inode': 675389768L, 'minor': 20}, 'version': 'v3_8 CVS $Revision: 1.963 $ encp.pyc', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/data/data2/Migration_Spool/VO6153:0000_000000000_0032300', 'status': ('ok', None), 'outfile': '/pnfs/fs/usr/Migration/ktev/migrated_from_dlt/KQ/KQRC/KQRC52/.(access)(00280000000000000841A148)', 'fc': {'size': 477904L, 'external_label': 'VOV138', 'pnfsid': '00280000000000000841A148', 'address': ('131.225.13.59', 7501)}, 'infilepath': '/data/data2/Migration_Spool/VO6153:0000_000000000_0032300', 'at_the_top': 42, 'local_inode': 4495347L, 'resend': {'max_retry': 3, 'resubmits': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.13.19', 57442), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'lm_dequeued': 1257314417.392499, 'encp_start_time': 1257314403.944859, 'job_queued': 1257314404.256093, 't0': 1257314403, 'in_queue': 13.136147975921631}, 'user_level': 'admin', 'unique_id': 'stkendm7a.fnal.gov-1257314404-18643-24632'} Thread client_connect_thread 00:00:17 stkendm7a.fnal.gov 018643 root I ENCP Waiting for mover to connect control socket at address: ('131.225.13.19', 57442) elapsed=154663.850sec Thread WRITE_2 00:00:17 stkendm7a.fnal.gov 018643 root I ENCP Control socket ('131.225.13.19', 57442) is connected to ('131.225.215.76', 60657) for stkendm7a.fnal.gov-1257314404-18643-24632. elapsed=154663.850sec Thread WRITE_2 00:00:17 stkendm7a.fnal.gov 018643 root I ENCP Received callback ticket from mover LTO4_92.mover for transfer stkendm7a.fnal.gov-1257314404-18643-24632. elapsed=154663.860sec Thread WRITE_2 00:00:17 stkendm7a.fnal.gov 018643 root I ENCP Attempting to connect data socket to mover at ('131.225.215.76', 60656). Thread WRITE_2 00:00:17 stkendm7a.fnal.gov 018643 root I ENCP Connecting data socket to mover (('131.225.215.76', 60656)) with interface 131.225.13.19. elapsed=154663.860sec Thread WRITE_2 00:00:17 stkendm7a.fnal.gov 018643 root I ENCP Data socket ('131.225.13.19', 57446) is connected to ('131.225.215.76', 60656). elapsed=154663.860sec Thread WRITE_2 00:00:17 stkendm7a.fnal.gov 018643 root I ENCP Opening local file. Thread WRITE_2 00:00:17 stkendm7a.fnal.gov 018643 root I ENCP pf_status = ('ok', None) skip_layer_cleanup = False Thread WRITE_2 00:00:17 cdfenmvr192a.fnal.gov 006151 root I LTO4_92MV tape position: target 5182 current 5182 0 Thread seek_thread 00:00:17 cdfenmvr192a.fnal.gov 006151 root I LTO4_92MV write_tape starting, bytes_to_write=478208 Thread tape_thread 00:00:17 cdfenmvr192a.fnal.gov 006151 root I LTO4_92MV Write starting. Tape VOV138 absolute location in blocks 255455 Thread tape_thread 00:00:17 stkendm7a.fnal.gov 018643 root I ENCP File /data/data2/Migration_Spool/VO6153:0000_000000000_0032300 transfered for stkendm7a.fnal.gov-1257314404-18643-24632 in 0.0089271068573 seconds. Thread WRITE_2 00:00:17 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:17 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM write_to_hsm next work on vol=VOS019 mover=LTO4_96.mover requester:root 00:00:17 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM HAVE_BOUND:sending write_to_hsm /pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0082/AE3F29DF-CEC8-DE11-B224-003048678ED4.root to mover LTO4_96.mover ('131.225.13.191', 7857) DEL_DISM None 00:00:17 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM MSG_TYPE=ADD_TO_LMQ request added to work at movers queue for storage group : cms 00:00:17 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM HAVE_BOUND:sending {'vc': {'storage_group': 'cms', 'external_label': 'VOS019', 'library': 'CD-LTO4G1', 'file_family': 'cms11', 'wrapper': 'cpio_odc', 'address': ('131.225.13.59', 7502), 'volume_family': 'cms.cms11.cpio_odc', 'file_family_width': 10}, 'outfilepath': '/pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0082/AE3F29DF-CEC8-DE11-B224-003048678ED4.root', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 25, 'delayed_dismount': None, 'curpri': 25}, 'file_size': 2412615773L, 'mover': 'LTO4_96.mover', 'ignore_fair_share': None, 'retry': None, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0082/AE3F29DF-CEC8-DE11-B224-003048678ED4.root', 'uid': 0, 'uname': 'root', 'type': 'cpio_odc', 'gid': 0, 'mtime': 1257312802, 'rmajor': 0, 'machine': ('Linux', 'cmsstor147.fnal.gov', '2.6.31', '#1 SMP Mon Sep 14 09:43:20 CDT 2009', 'x86_64'), 'sanity_size': 65536, 'pstat': (16893, 120974272, 23L, 1, 9811, 5063, 512, 1257292920, 1257292920, 1257286060), 'mode_octal': '0100036', 'mode': 33188, 'gname': 'root', 'size_bytes': 2412615773L, 'fullname': '/storage/data2/write-pool-2/data/0007000000000000623CD5B0', 'inode': 121427376L, 'minor': 23}, 'version': 'v3_7d CVS $Revision: 1.922 $ encp', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/storage/data2/write-pool-2/data/0007000000000000623CD5B0', 'status': ('ok', None), 'outfile': '/pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0082/.(access)(0007000000000000623CD5B0)', 'fc': {'size': 2412615773L, 'external_label': 'VOS019', 'address': ('131.225.13.59', 7501)}, 'infilepath': '/storage/data2/write-pool-2/data/0007000000000000623CD5B0', 'at_the_top': 44, 'local_inode': 7201, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.189.118', 33026), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'lm_dequeued': 1257314417.608731, 'encp_start_time': 1257312801.1664889, 'job_queued': 1257312802.1578419, 't0': 1257312801, 'in_queue': 1615.4505820274353}, 'user_level': 'admin', 'unique_id': 'cmsstor147.fnal.gov-1257312802-2565-0'} to mover 00:00:17 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:17 stkenmvr196a.fnal.gov 027637 root I LTO4_96MV WRITE_TO_HSM Thread MainThread 00:00:17 stkenmvr196a.fnal.gov 027637 root I LTO4_96MV SENDING {'vc': {'storage_group': 'cms', 'external_label': 'VOS019', 'library': 'CD-LTO4G1', 'file_family': 'cms11', 'wrapper': 'cpio_odc', 'address': ('131.225.13.59', 7502), 'volume_family': 'cms.cms11.cpio_odc', 'file_family_width': 10}, 'outfilepath': '/pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0082/AE3F29DF-CEC8-DE11-B224-003048678ED4.root', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 25, 'delayed_dismount': None, 'curpri': 25}, 'file_size': 2412615773L, 'mover': {'max_buffer': 2621440000L, 'check_written_file': 600, 'max_failures': 5, 'library': 'CD-LTO4G1.library_manager', 'local_mover': 0, 'single_filemark': 1, 'max_consecutive_failures': 3, 'norestart': 'INQ', 'port': 7857, 'mc_device': '0,7,1,2', 'compression': 0, 'serial_num': '9310108548', 'update_interval_in_bound': 2, 'update_interval': 5, 'syslog_entry': 'sense|st[0-9]', 'status': ('ok', None), 'max_rate': 73400320.0, 'vendor_id': 'IBM', 'driver': 'FTTDriver', 'host': 'stkenmvr196a', 'max_time_in_state': 1200, 'device': 'stkenmvr196a:/dev/rmt/tps0d0n', 'name': 'LTO4_96.mover', 'send_stats': 1, 'product_id': 'ULTRIUM-TD4', 'statistics_path': '/tmp/enstore/enstore/LTO4_96.stat', 'mover_address': ('131.225.13.191', 7857), 'mount_delay': 15, 'check_first_written_file': 0, 'callback_addr': ('131.225.13.191', 38300), 'data_ip': 'stkenmvr196a', 'logname': 'LTO4_96MV', 'hostip': '131.225.13.191', 'do_cleaning': 'No', 'media_changer': 'SL8500G1.media_changer'}, 'ignore_fair_share': None, 'retry': None, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0082/AE3F29DF-CEC8-DE11-B224-003048678ED4.root', 'uid': 0, 'uname': 'root', 'mtime': 1257312802, 'rmajor': 0, 'machine': ('Linux', 'cmsstor147.fnal.gov', '2.6.31', '#1 SMP Mon Sep 14 09:43:20 CDT 2009', 'x86_64'), 'sanity_size': 65536, 'type': 'cpio_odc', 'pstat': (16893, 120974272, 23L, 1, 9811, 5063, 512, 1257292920, 1257292920, 1257286060), 'mode_octal': '0100036', 'mode': 33188, 'size_bytes': 2412615773L, 'gname': 'root', 'gid': 0, 'fullname': '/storage/data2/write-pool-2/data/0007000000000000623CD5B0', 'inode': 121427376L, 'minor': 23}, 'version': 'v3_7d CVS $Revision: 1.922 $ encp', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/storage/data2/write-pool-2/data/0007000000000000623CD5B0', 'status': ('ok', None), 'outfile': '/pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0082/.(access)(0007000000000000623CD5B0)', 'fc': {'size': 2412615773L, 'external_label': 'VOS019', 'address': ('131.225.13.59', 7501)}, 'infilepath': '/storage/data2/write-pool-2/data/0007000000000000623CD5B0', 'at_the_top': 44, 'local_inode': 7201, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.189.118', 33026), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'lm_dequeued': 1257314417.608731, 'encp_start_time': 1257312801.1664889, 'job_queued': 1257312802.1578419, 't0': 1257312801, 'in_queue': 1615.4505820274353}, 'user_level': 'admin', 'unique_id': 'cmsstor147.fnal.gov-1257312802-2565-0'} Thread client_connect_thread 00:00:17 cmsstor147.fnal.gov 002565 root I ENCP Control socket ('131.225.189.118', 33026) is connected to ('131.225.13.191', 38301) for cmsstor147.fnal.gov-1257312802-2565-0. elapsed=1616.300sec 00:00:17 cmsstor147.fnal.gov 002565 root I ENCP Received callback ticket from mover LTO4_96.mover for transfer cmsstor147.fnal.gov-1257312802-2565-0. elapsed=1616.300sec 00:00:17 cmsstor147.fnal.gov 002565 root I ENCP Atempting to connect data socket to mover at ('131.225.13.191', 38300). 00:00:17 cmsstor147.fnal.gov 002565 root I ENCP Connecting data socket to mover (('131.225.13.191', 38300)) with interface 131.225.189.118. elapsed=1616.300sec 00:00:17 cmsstor147.fnal.gov 002565 root I ENCP Data socket ('131.225.189.118', 60310) is connected to ('131.225.13.191', 38300). elapsed=1616.310sec 00:00:17 stkendm7a.fnal.gov 018643 root I ENCP Version: v3_8 CVS $Revision: 1.963 $ encp.pyc OS: Linux 2.6.9-89.0.9.ELsmp i686 Release: Scientific Linux Fermi LTS release 4.4 (Wilson) User: root(0) Group: root(0) Euser: root(0) Egroup: root(0) Library: CD-LTO4F1 Storage Group: ktev File Family: ktev_migration-MIGRATION FF Wrapper: cern FF Width: 1 Current working directory: stkendm7a.fnal.gov:/opt/enstore/src Command line: encp --delayed-dismount 2 --ignore-fair-share --threaded --priority 0 --storage-group ktev --file-family ktev_migration-MIGRATION --file-family-wrapper cern --library CD-LTO4F1 --override-path /pnfs/fs/usr/ktev/migrated_from_dlt/KQ/KQRC/KQRC42/f1909 --threaded /data/data2/Migration_Spool/VO6153:0000_000000000_0032558 /pnfs/fs/usr/Migration/ktev/migrated_from_dlt/KQ/KQRC/KQRC42/f1909 Thread WRITE_1 00:00:17 cmsstor147.fnal.gov 002565 root I ENCP Opening local file. 00:00:17 stkenmvr196a.fnal.gov 027637 root I LTO4_96MV tape position: target 82 current 82 0 Thread seek_thread 00:00:17 stkenmvr196a.fnal.gov 027637 root I LTO4_96MV write_tape starting, bytes_to_write=2412616192 Thread tape_thread 00:00:17 stkendm7a.fnal.gov 018643 root I ENCP Creating zero length output files. Thread WRITE_1 00:00:17 stkenmvr196a.fnal.gov 027637 root I LTO4_96MV Write starting. Tape VOS019 absolute location in blocks 1858507 Thread tape_thread 00:00:17 cmsstor147.fnal.gov 002565 root I ENCP pf_status = ('ok', None) skip_layer_cleanup = False 00:00:17 stkendm7a.fnal.gov 018643 root I ENCP Sending /pnfs/fs/usr/Migration/ktev/migrated_from_dlt/KQ/KQRC/KQRC42/f1909 write request to LM: unique_id: stkendm7a.fnal.gov-1257314417-18643-24634 inputfile: /data/data2/Migration_Spool/VO6153:0000_000000000_0032558 outputfile: /pnfs/fs/usr/Migration/ktev/migrated_from_dlt/KQ/KQRC/KQRC42/.(access)(00280000000000000841A178) Thread WRITE_1 00:00:17 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM MSG_TYPE=ADD_TO_LMQ request added to pending queue for storage group : ktev 00:00:17 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM write Q'd /data/data2/Migration_Spool/VO6153:0000_000000000_0032558 (stkendm7a.fnal.gov-1257314417-18643-24634) -> /pnfs/fs/usr/ktev/migrated_from_dlt/KQ/KQRC/KQRC42/f1909 : library=CD-LTO4F1 family=ktev_migration-MIGRATION requester:root volume_family:ktev.ktev_migration-MIGRATION.cern 00:00:18 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM 00:00:18 stkenmvr202a.fnal.gov 006121 root I LTO4_102MV written bytes 2354424832/2354424832, blocks 17963 header 192 trailer 155 Thread tape_thread 00:00:18 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM 00:00:18 stkensrv4n.fnal.gov 004406 enstore I SL8500G1MC query drive 0,5,1,15 => 0,0,5,1,15 online available IBM-LTO4 00:00:18 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:18 cmsstor65.fnal.gov 029700 root I ENCP File /storage/data2/write-pool-2/data/0007000000000000623EF7E8 transfered for cmsstor65.fnal.gov-1257312771-29700-0 in 25.2080760002 seconds. 00:00:18 cmsstor153.fnal.gov 015719 root I ENCP Version: v3_7d CVS $Revision: 1.922 $ encp OS: Linux 2.6.31 x86_64 Release: Rocks release 5.1 (V.I) Scientific Linux SL release 5.2 (Boron) User: root(0) Group: root(0) Euser: root(0) Egroup: root(0) Library: CD-LTO4G1 Storage Group: cms File Family: cms11 FF Wrapper: cpio_odc FF Width: 10 Current working directory: cmsstor153.fnal.gov:/root Command line: /home/enstore/enstore/bin/encp --verbose 4 --threaded --ecrc --resubmit-timeout 1800 --priority 25 --pnfs-mount /pnfs/fs --put-cache 00070000000000006253AA78 /storage/data3/write-pool-3/data/00070000000000006253AA78 00:00:18 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:18 stkensrv4n.fnal.gov 005843 enstore I SL8500F1MC FINISHED listDrives returned ('ok', 0, None, '', '') 00:00:18 stkensrv0n.fnal.gov 023312 enstore I INFSRV stop listing all volumes (3) 00:00:19 cmsstor153.fnal.gov 015719 root I ENCP Creating zero length output files. 00:00:19 cmsstor153.fnal.gov 015719 root I ENCP Sending /pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0086/CCEFF339-EFC8-DE11-AAA9-003048678B18.root write request to LM: unique_id: cmsstor153.fnal.gov-1257314418-15719-0 inputfile: /storage/data3/write-pool-3/data/00070000000000006253AA78 outputfile: /pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0086/.(access)(00070000000000006253AA78) 00:00:19 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM MSG_TYPE=ADD_TO_LMQ request added to pending queue for storage group : cms 00:00:19 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM write Q'd /storage/data3/write-pool-3/data/00070000000000006253AA78 (cmsstor153.fnal.gov-1257314418-15719-0) -> /pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0086/CCEFF339-EFC8-DE11-AAA9-003048678B18.root : library=CD-LTO4G1 family=cms11 requester:root volume_family:cms.cms11.cpio_odc 00:00:19 cmsstor153.fnal.gov 015719 root I ENCP Listening for control socket at: ('131.225.189.124', 54611) 00:00:19 cmsstor153.fnal.gov 015719 root I ENCP Waiting for mover to connect control socket at address: ('131.225.189.124', 54611) elapsed=1.020sec 00:00:19 cdfenmvr192a.fnal.gov 006151 root I LTO4_92MV written bytes 478208/478208, blocks 4 header 0 trailer 304 Thread tape_thread 00:00:19 stkenmvr202a.fnal.gov 006121 root I LTO4_102MV filemarks written. Tape VOS315 absolute location in blocks 871356 Thread tape_thread 00:00:19 stkenmvr202a.fnal.gov 006121 root I LTO4_102MV write_tape timing:pull 0.229344129562 write 7.66966128349 crc_check 10.927164793 freespace 0.200905561447 block_write 19.0270757675 idle 0.350483417511 Thread tape_thread 00:00:19 stkenmvr202a.fnal.gov 006121 root I LTO4_102MV drive stats after write. Tape VOS315 position 34 block 0 block_size 0 bloc_loc 871356 tot_blocks 762939000 BOT 0 read_err 0 write_err 2 bytes 2354424832 block_write_tot 19.0270757675 tape_rate 123740760.838 Thread tape_thread 00:00:19 stkenmvr202a.fnal.gov 006121 root I LTO4_102MV remaining bytes info in DB 671767552000 estimated 669413127168 reported from drive 669569024000 position 35 Thread tape_thread 00:00:19 stkenmvr202a.fnal.gov 006121 root I LTO4_102MV new bitfile request {'uid': 0, 'external_label': 'VOS315', 'gid': 0, 'location_cookie': '0000_000000000_0000034', 'complete_crc': 3928670495L, 'sanity_cookie': (65536L, 4063849046L), 'size': 2354424485L} Thread tape_thread 00:00:19 cmsstor82.fnal.gov 014013 root I ENCP File /storage/data3/write-pool-3/data/000700000000000062228078 transfered for cmsstor82.fnal.gov-1257312754-14013-0 in 38.2288248539 seconds. 00:00:19 stkenmvr202a.fnal.gov 006121 root I LTO4_102MV set remaining: VOS315 669569024000 0000_000000000_0000035 Thread tape_thread 00:00:19 last message repeated 1 times 00:00:19 stkensrv0n.fnal.gov 028762 enstore I VOLSRV Updating update volume set last_access = '2009-11-04 00:00:19', eod_cookie = '0000_000000000_0000035', non_del_files = '34', remaining_bytes = 669569024000 where label = 'VOS315'; 00:00:19 stkensrv4n.fnal.gov 003721 enstore I SL8500G1MC mount VON129 0,5,1,14 => 0,Mount: VON129 mounted on 0, 5, 1,14 00:00:20 stkensrv4n.fnal.gov 003689 enstore I SL8500G1MC FINISHED mount VON129 0,5,1,14 returned ('ok', 0, 'mount VON129 0,5,1,14 => 0,Mount: VON129 mounted on 0, 5, 1,14', '', '') 00:00:20 stkenmvr202a.fnal.gov 006121 root I LTO4_102MV transfer complete volume=VOS315 location=35 Thread tape_thread 00:00:20 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM 00:00:20 last message repeated 1 times 00:00:20 stkensrv0n.fnal.gov 028762 enstore I VOLSRV Updating update volume set sum_wr_access = '34', last_access = '2009-11-04 00:00:20', modification_time = '2009-11-04 00:00:20' where label = 'VOS315'; 00:00:20 stkenmvr202a.fnal.gov 006121 root I LTO4_102MV Sending done to client: {'vc': {'comment': '', 'storage_group': 'cms', 'blocksize': 131072, 'declared': 1255359578.0, 'library': 'CD-LTO4G1', 'si_time': [0.0, 1251004349.0], 'user_inhibit': ['none', 'none'], 'system_inhibit': ['none', 'none'], 'external_label': 'VOS315', 'wrapper': 'cpio_odc', 'remaining_bytes': 671767552000L, 'sum_mounts': 47, 'sum_rd_access': 0, 'file_family_width': 10, 'last_access': 1257314393.0, 'media_type': 'LTO4', 'status': ('ok', None), 'eod_cookie': '0000_000000000_0000034', 'capacity_bytes': 858993459200L, 'non_del_files': 33, 'sum_wr_err': 0, 'sum_wr_access': 33, 'file_family': 'cms11', 'address': ('131.225.13.59', 7502), 'volume_family': 'cms.cms11.cpio_odc', 'modification_time': 1257314393.0, 'write_protected': 'n', 'sum_rd_err': 0, 'first_access': 1257312793.0}, 'outfilepath': '/pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0081/34BBF568-C2C8-DE11-BB7C-0030486792DE.root', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 25, 'delayed_dismount': None, 'curpri': 25}, 'file_size': 2354424485L, 'mover': {'max_buffer': 2621440000L, 'check_written_file': 600, 'max_failures': 5, 'library': 'CD-LTO4G1.library_manager', 'local_mover': 0, 'single_filemark': 1, 'max_consecutive_failures': 3, 'norestart': 'INQ', 'port': 7862, 'mc_device': '0,6,1,14', 'compression': 0, 'serial_num': '1310051194', 'update_interval_in_bound': 2, 'update_interval': 5, 'syslog_entry': 'sense|st[0-9]', 'status': ('ok', None), 'max_rate': 73400320.0, 'vendor_id': 'IBM', 'driver': 'FTTDriver', 'host': 'stkenmvr202a', 'max_time_in_state': 1200, 'device': 'stkenmvr202a:/dev/rmt/tps0d0n', 'name': 'LTO4_102.mover', 'send_stats': 1, 'product_id': 'ULTRIUM-TD4', 'statistics_path': '/tmp/enstore/enstore/LTO4_102.stat', 'mover_address': ('131.225.13.197', 7862), 'mount_delay': 15, 'check_first_written_file': 0, 'callback_addr': ('131.225.13.197', 38265), 'data_ip': 'stkenmvr202a', 'logname': 'LTO4_102MV', 'hostip': '131.225.13.197', 'do_cleaning': 'No', 'media_changer': 'SL8500G1.media_changer'}, 'ignore_fair_share': None, 'retry': None, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0081/34BBF568-C2C8-DE11-BB7C-0030486792DE.root', 'uid': 0, 'uname': 'root', 'mtime': 1257312779, 'rmajor': 0, 'machine': ('Linux', 'cmsstor81.fnal.gov', '2.6.31', '#1 SMP Mon Sep 14 09:43:20 CDT 2009', 'x86_64'), 'sanity_size': 65536, 'type': 'cpio_odc', 'pstat': (16893, 120498856, 21L, 1, 9811, 5063, 512, 1257293305, 1257293305, 1257282260), 'mode_octal': '0100036', 'mode': 33188, 'size_bytes': 2354424485L, 'gname': 'root', 'gid': 0, 'fullname': '/storage/data3/write-pool-3/data/000700000000000062342618', 'inode': 120858136L, 'minor': 21}, 'version': 'v3_7d CVS $Revision: 1.922 $ encp', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/storage/data3/write-pool-3/data/000700000000000062342618', 'status': ('ok', None), 'outfile': '/pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0081/.(access)(000700000000000062342618)', 'fc': {'uid': 0, 'external_label': 'VOS315', 'gid': 0, 'location_cookie': '0000_000000000_0000034', 'complete_crc': 3928670495L, 'bfid': 'CDMS125731441900000', 'sanity_cookie': (65536L, 4063849046L), 'size': 2354424485L}, 'infilepath': '/storage/data3/write-pool-3/data/000700000000000062342618', 'at_the_top': 2, 'local_inode': 1073745557, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.205.203', 47489), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'encp_start_time': 1257312778.6277161, 'seek_time': 0.029968976974487305, 'transfer_time': 25.816561937332153, 't0': 1257312778, 'mount_time': 0.014426946640014648, 'lm_dequeued': 1257314394.050235, 'job_queued': 1257312779.85356, 'drive_call_time': 7.415886402130127, 'in_queue': 1614.1963639259338, 'drive_transfer_time': 21.153100967407227}, 'user_level': 'admin', 'unique_id': 'cmsstor81.fnal.gov-1257312779-15087-0'} Thread tape_thread 00:00:20 last message repeated 1 times 00:00:20 stkenmvr202a.fnal.gov 006121 root I LTO4_102MV Done is sent Thread tape_thread 00:00:20 stkenmvr202a.fnal.gov 006121 root I LTO4_102MV maybe_clean Thread tape_thread 00:00:20 last message repeated 1 times 00:00:20 stkensrv0n.fnal.gov 028762 enstore I VOLSRV Updating update volume set sum_mounts = '195', last_access = '2009-11-04 00:00:20' where label = 'VON129'; 00:00:20 stkenmvr202a.fnal.gov 006121 root I LTO4_102MV returned from maybe_clean Thread tape_thread 00:00:20 cmsstor81.fnal.gov 015087 root I ENCP Received final dialog for cmsstor81.fnal.gov-1257312779-15087-0. elapsed=1641.590sec 00:00:20 stkenmvr210a.fnal.gov 011436 root I LTO4_110MV MSG_TYPE=MC_LOAD_DONE mounted VON129 ULTRIUM-TD4 00:00:20 Thread media_thread 00:00:20 cmsstor81.fnal.gov 015087 root I ENCP leaving transfer_file(). 00:00:20 cmsstor81.fnal.gov 015087 root I ENCP pf_status = ('ok', None) skip_layer_cleanup = False 00:00:20 stkensrv0n.fnal.gov 028796 enstore I FILSRV Updating update file set deleted = 'n', pnfs_path = '/pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0081/34BBF568-C2C8-DE11-BB7C-0030486792DE.root', drive = 'stkenmvr202a:/dev/rmt/tps0d0n:1310051194', pnfs_id = '000700000000000062342618' where bfid = 'CDMS125731441900000'; 00:00:20 cmsstor81.fnal.gov 015087 root I ENCP MSG_TYPE=ENCP_XFER write_to_hsm /storage/data3/write-pool-3/data/000700000000000062342618 -> /pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0081/34BBF568-C2C8-DE11-BB7C-0030486792DE.root: 2354424485 bytes copied to VOS315 at 86.9 MB/S (88.4 MB/S network) (106 MB/S drive) (109 MB/S disk) mover=LTO4_102.mover drive_id=ULTRIUM-TD4 drive_sn=1310051194 drive_vendor=IBM elapsed=1641.8 {'storage_group': 'cms', 'encp_crc': 3928670495L, 'disk_rate': 108.73912889913088, 'drive_rate': 106.14775416248246, 'driver': 'FTTDriver', 'transfer_rate': 86.905972924977448, 'mover_interface': 'stkenmvr202a', 'overall_rate': 85.663031756135197, 'media_changer': 'SL8500G1.media_changer', 'network_rate': 86.905972924977448, 'encp_ip': '131.225.205.203', 'unique_id': 'cmsstor81.fnal.gov-1257312779-15087-0'} 00:00:20 cmsstor81.fnal.gov 015087 root I ENCP File /pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0081/34BBF568-C2C8-DE11-BB7C-0030486792DE.root copied successfully. 00:00:20 cmsstor81.fnal.gov 015087 root I ENCP Completed transferring 2354424485 bytes in 1 files in 1641.77302384 sec. Overall rate = 85.7 MB/sec. Transfer rate = 86.9 MB/sec. Network rate = 88.4 MB/sec. Drive rate = 106 MB/sec. Disk rate = 109 MB/sec. Exit status = 0. 00:00:20 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:21 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:21 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM write_to_hsm next work on vol=VOS315 mover=LTO4_102.mover requester:root 00:00:21 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM HAVE_BOUND:sending write_to_hsm /pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0085/F06323E6-DFC8-DE11-AF93-003048D25B68.root to mover LTO4_102.mover ('131.225.13.197', 7862) DEL_DISM None 00:00:21 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM MSG_TYPE=ADD_TO_LMQ request added to work at movers queue for storage group : cms 00:00:21 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM HAVE_BOUND:sending {'vc': {'storage_group': 'cms', 'external_label': 'VOS315', 'library': 'CD-LTO4G1', 'file_family': 'cms11', 'wrapper': 'cpio_odc', 'address': ('131.225.13.59', 7502), 'volume_family': 'cms.cms11.cpio_odc', 'file_family_width': 10}, 'outfilepath': '/pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0085/F06323E6-DFC8-DE11-AF93-003048D25B68.root', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 25, 'delayed_dismount': None, 'curpri': 25}, 'file_size': 2255823047L, 'mover': 'LTO4_102.mover', 'ignore_fair_share': None, 'retry': None, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0085/F06323E6-DFC8-DE11-AF93-003048D25B68.root', 'uid': 0, 'uname': 'root', 'type': 'cpio_odc', 'gid': 0, 'mtime': 1257312803, 'rmajor': 0, 'machine': ('Linux', 'cmsstor95.fnal.gov', '2.6.31', '#1 SMP Mon Sep 14 09:43:20 CDT 2009', 'x86_64'), 'sanity_size': 65536, 'pstat': (16893, 122155992, 23L, 1, 9811, 5063, 512, 1257304578, 1257304578, 1257296604), 'mode_octal': '0100036', 'mode': 33188, 'gname': 'root', 'size_bytes': 2255823047L, 'fullname': '/storage/data3/write-pool-3/data/0007000000000000624A4340', 'inode': 122307392L, 'minor': 23}, 'version': 'v3_7d CVS $Revision: 1.922 $ encp', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/storage/data3/write-pool-3/data/0007000000000000624A4340', 'status': ('ok', None), 'outfile': '/pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0085/.(access)(0007000000000000624A4340)', 'fc': {'size': 2255823047L, 'external_label': 'VOS315', 'address': ('131.225.13.59', 7501)}, 'infilepath': '/storage/data3/write-pool-3/data/0007000000000000624A4340', 'at_the_top': 6, 'local_inode': 1073744596, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.205.217', 50966), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'lm_dequeued': 1257314421.0814781, 'encp_start_time': 1257312802.3007939, 'job_queued': 1257312803.696573, 't0': 1257312802, 'in_queue': 1617.384584903717}, 'user_level': 'admin', 'unique_id': 'cmsstor95.fnal.gov-1257312803-10575-0'} to mover 00:00:21 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:21 stkenmvr202a.fnal.gov 006121 root I LTO4_102MV WRITE_TO_HSM Thread MainThread 00:00:21 stkenmvr202a.fnal.gov 006121 root I LTO4_102MV SENDING {'vc': {'storage_group': 'cms', 'external_label': 'VOS315', 'library': 'CD-LTO4G1', 'file_family': 'cms11', 'wrapper': 'cpio_odc', 'address': ('131.225.13.59', 7502), 'volume_family': 'cms.cms11.cpio_odc', 'file_family_width': 10}, 'outfilepath': '/pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0085/F06323E6-DFC8-DE11-AF93-003048D25B68.root', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 25, 'delayed_dismount': None, 'curpri': 25}, 'file_size': 2255823047L, 'mover': {'max_buffer': 2621440000L, 'check_written_file': 600, 'max_failures': 5, 'library': 'CD-LTO4G1.library_manager', 'local_mover': 0, 'single_filemark': 1, 'max_consecutive_failures': 3, 'norestart': 'INQ', 'port': 7862, 'mc_device': '0,6,1,14', 'compression': 0, 'serial_num': '1310051194', 'update_interval_in_bound': 2, 'update_interval': 5, 'syslog_entry': 'sense|st[0-9]', 'status': ('ok', None), 'max_rate': 73400320.0, 'vendor_id': 'IBM', 'driver': 'FTTDriver', 'host': 'stkenmvr202a', 'max_time_in_state': 1200, 'device': 'stkenmvr202a:/dev/rmt/tps0d0n', 'name': 'LTO4_102.mover', 'send_stats': 1, 'product_id': 'ULTRIUM-TD4', 'statistics_path': '/tmp/enstore/enstore/LTO4_102.stat', 'mover_address': ('131.225.13.197', 7862), 'mount_delay': 15, 'check_first_written_file': 0, 'callback_addr': ('131.225.13.197', 38267), 'data_ip': 'stkenmvr202a', 'logname': 'LTO4_102MV', 'hostip': '131.225.13.197', 'do_cleaning': 'No', 'media_changer': 'SL8500G1.media_changer'}, 'ignore_fair_share': None, 'retry': None, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0085/F06323E6-DFC8-DE11-AF93-003048D25B68.root', 'uid': 0, 'uname': 'root', 'mtime': 1257312803, 'rmajor': 0, 'machine': ('Linux', 'cmsstor95.fnal.gov', '2.6.31', '#1 SMP Mon Sep 14 09:43:20 CDT 2009', 'x86_64'), 'sanity_size': 65536, 'type': 'cpio_odc', 'pstat': (16893, 122155992, 23L, 1, 9811, 5063, 512, 1257304578, 1257304578, 1257296604), 'mode_octal': '0100036', 'mode': 33188, 'size_bytes': 2255823047L, 'gname': 'root', 'gid': 0, 'fullname': '/storage/data3/write-pool-3/data/0007000000000000624A4340', 'inode': 122307392L, 'minor': 23}, 'version': 'v3_7d CVS $Revision: 1.922 $ encp', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/storage/data3/write-pool-3/data/0007000000000000624A4340', 'status': ('ok', None), 'outfile': '/pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0085/.(access)(0007000000000000624A4340)', 'fc': {'size': 2255823047L, 'external_label': 'VOS315', 'address': ('131.225.13.59', 7501)}, 'infilepath': '/storage/data3/write-pool-3/data/0007000000000000624A4340', 'at_the_top': 6, 'local_inode': 1073744596, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.205.217', 50966), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'lm_dequeued': 1257314421.0814781, 'encp_start_time': 1257312802.3007939, 'job_queued': 1257312803.696573, 't0': 1257312802, 'in_queue': 1617.384584903717}, 'user_level': 'admin', 'unique_id': 'cmsstor95.fnal.gov-1257312803-10575-0'} Thread client_connect_thread 00:00:21 cmsstor95.fnal.gov 010575 root I ENCP Control socket ('131.225.205.217', 50966) is connected to ('131.225.13.197', 38268) for cmsstor95.fnal.gov-1257312803-10575-0. elapsed=1619.020sec 00:00:21 cmsstor95.fnal.gov 010575 root I ENCP Received callback ticket from mover LTO4_102.mover for transfer cmsstor95.fnal.gov-1257312803-10575-0. elapsed=1619.020sec 00:00:21 cmsstor95.fnal.gov 010575 root I ENCP Atempting to connect data socket to mover at ('131.225.13.197', 38267). 00:00:21 cmsstor95.fnal.gov 010575 root I ENCP Connecting data socket to mover (('131.225.13.197', 38267)) with interface 131.225.205.217. elapsed=1619.020sec 00:00:21 cmsstor95.fnal.gov 010575 root I ENCP Data socket ('131.225.205.217', 60516) is connected to ('131.225.13.197', 38267). elapsed=1619.020sec 00:00:21 stkenmvr202a.fnal.gov 006121 root I LTO4_102MV tape position: target 35 current 35 0 Thread seek_thread 00:00:21 stkenmvr202a.fnal.gov 006121 root I LTO4_102MV write_tape starting, bytes_to_write=2255823360 Thread tape_thread 00:00:21 cmsstor95.fnal.gov 010575 root I ENCP Opening local file. 00:00:21 stkenmvr202a.fnal.gov 006121 root I LTO4_102MV Write starting. Tape VOS315 absolute location in blocks 871356 Thread tape_thread 00:00:21 cmsstor95.fnal.gov 010575 root I ENCP pf_status = ('ok', None) skip_layer_cleanup = False 00:00:21 cmsstor79.fnal.gov 006780 root I ENCP File /storage/data2/write-pool-2/data/000700000000000062604C70 transfered for cmsstor79.fnal.gov-1257314142-6780-0 in 79.8373138905 seconds. 00:00:22 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM 00:00:22 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:22 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:23 cmsstor118.fnal.gov 013119 root I ENCP File /storage/data1/write-pool-1/data/000700000000000062535DD0 transfered for cmsstor118.fnal.gov-1257312759-13119-0 in 39.1448431015 seconds. 00:00:23 cmsstor233.fnal.gov 004201 root I ENCP Version: v3_7d CVS $Revision: 1.922 $ encp OS: Linux 2.6.31 x86_64 Release: Rocks release 5.1 (V.I) Scientific Linux SL release 5.2 (Boron) User: root(0) Group: root(0) Euser: root(0) Egroup: root(0) Library: CD-LTO4G1 Storage Group: cms File Family: cms11 FF Wrapper: cpio_odc FF Width: 10 Current working directory: cmsstor233.fnal.gov:/root Command line: /home/enstore/enstore/bin/encp --verbose 4 --threaded --ecrc --resubmit-timeout 1800 --priority 25 --pnfs-mount /pnfs/fs --put-cache 00070000000000006253FB80 /storage/data1/write-pool-1/data/00070000000000006253FB80 00:00:23 stkenmvr144a.fnal.gov 006329 root I LTO4_44MV written bytes 4115627520/4115627520, blocks 31400 header 192 trailer 444 Thread tape_thread 00:00:23 cdfenmvr192a.fnal.gov 006151 root I LTO4_92MV filemarks written. Tape VOV138 absolute location in blocks 255464 Thread tape_thread 00:00:23 cdfenmvr192a.fnal.gov 006151 root I LTO4_92MV write_tape timing:pull 3.2901763916e-05 write 0.00531601905823 crc_check 0.00246000289917 freespace 2.90870666504e-05 block_write 0.00783801078796 idle 6.60419464111e-05 Thread tape_thread 00:00:23 cdfenmvr192a.fnal.gov 006151 root I LTO4_92MV drive stats after write. Tape VOV138 position 5182 block 0 block_size 0 bloc_loc 255464 tot_blocks 762939000 BOT 0 read_err 0 write_err 14 bytes 478208 block_write_tot 0.00783801078796 tape_rate 61011398.547 Thread tape_thread 00:00:23 cdfenmvr192a.fnal.gov 006151 root I LTO4_92MV remaining bytes info in DB 738923520000 estimated 738923041792 reported from drive 738919424000 position 5185 Thread tape_thread 00:00:23 cdfenmvr192a.fnal.gov 006151 root I LTO4_92MV new bitfile request {'uid': 0, 'external_label': 'VOV138', 'gid': 0, 'location_cookie': '0000_000000000_0005183', 'complete_crc': 633698066L, 'sanity_cookie': (65536L, 1073862877L), 'size': 477904L} Thread tape_thread 00:00:23 cdfenmvr192a.fnal.gov 006151 root I LTO4_92MV set remaining: VOV138 738919424000 0000_000000000_0005185 Thread tape_thread 00:00:23 last message repeated 1 times 00:00:23 stkensrv0n.fnal.gov 028762 enstore I VOLSRV Updating update volume set last_access = '2009-11-04 00:00:23', eod_cookie = '0000_000000000_0005185', non_del_files = '1729', remaining_bytes = 738919424000 where label = 'VOV138'; 00:00:23 stkenmvr151a.fnal.gov 013228 root I LTO4_51MV written bytes 5912685568/5912685568, blocks 45111 header 202 trailer 475 Thread tape_thread 00:00:23 cdfenmvr192a.fnal.gov 006151 root I LTO4_92MV transfer complete volume=VOV138 location=5185 Thread tape_thread 00:00:23 last message repeated 1 times 00:00:23 stkensrv0n.fnal.gov 028762 enstore I VOLSRV Updating update volume set sum_wr_access = '1728', modification_time = '2009-11-04 00:00:23' where label = 'VOV138'; 00:00:23 cdfenmvr192a.fnal.gov 006151 root I LTO4_92MV Sending done to client: {'vc': {'comment': '', 'storage_group': 'ktev', 'blocksize': 131072, 'declared': 1254165875.0, 'library': 'CD-LTO4F1', 'si_time': [0.0, 0.0], 'user_inhibit': ['none', 'none'], 'system_inhibit': ['none', 'none'], 'external_label': 'VOV138', 'wrapper': 'cern', 'remaining_bytes': 738923520000L, 'sum_mounts': 1, 'sum_rd_access': 0, 'file_family_width': 1, 'last_access': 1257314416.0, 'media_type': 'LTO4', 'status': ('ok', None), 'eod_cookie': '0000_000000000_0005182', 'capacity_bytes': 858993459200L, 'non_del_files': 1728, 'sum_wr_err': 0, 'sum_wr_access': 1727, 'file_family': 'ktev_migration-MIGRATION', 'address': ('131.225.13.59', 7502), 'volume_family': 'ktev.ktev_migration-MIGRATION.cern', 'modification_time': 1257314416.0, 'write_protected': 'n', 'sum_rd_err': 0, 'first_access': 1257302588.0}, 'outfilepath': '/pnfs/fs/usr/Migration/ktev/migrated_from_dlt/KQ/KQRC/KQRC52/f5012', 'encp': {'agetime': 30, 'adminpri': 6, 'delpri': 1, 'basepri': 1000007, 'delayed_dismount': 2, 'curpri': 1000007}, 'file_size': 477904L, 'mover': {'max_buffer': 2621440000L, 'check_written_file': 600, 'max_failures': 5, 'library': 'CD-LTO4F1.library_manager', 'local_mover': 0, 'single_filemark': 1, 'max_consecutive_failures': 3, 'norestart': 'INQ', 'port': 7853, 'mc_device': '1,0,1,1', 'compression': 0, 'serial_num': '1310060800', 'update_interval_in_bound': 2, 'update_interval': 5, 'syslog_entry': 'sense|st[0-9]', 'status': ('ok', None), 'max_rate': 73400320.0, 'vendor_id': 'IBM', 'driver': 'FTTDriver', 'host': 'cdfenmvr192a', 'max_time_in_state': 1200, 'device': 'cdfenmvr192a:/dev/rmt/tps0d0n', 'name': 'LTO4_92.mover', 'send_stats': 1, 'product_id': 'ULTRIUM-TD4', 'statistics_path': '/tmp/enstore/enstore/LTO4_92.stat', 'mover_address': ('131.225.215.76', 7853), 'mount_delay': 15, 'check_first_written_file': 0, 'callback_addr': ('131.225.215.76', 60656), 'data_ip': 'cdfenmvr192a', 'logname': 'LTO4_92MV', 'hostip': '131.225.215.76', 'do_cleaning': 'No', 'media_changer': 'SL8500F1.media_changer'}, 'ignore_fair_share': 1, 'retry': None, 'lm': {'address': ('131.225.13.4', 8542)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fs/usr/ktev/migrated_from_dlt/KQ/KQRC/KQRC52/f5012', 'uid': 0, 'uname': 'root', 'mtime': 1257314404, 'rmajor': 0, 'machine': ('Linux', 'stkendm7a.fnal.gov', '2.6.9-89.0.9.ELsmp', '#1 SMP Mon Aug 24 13:47:43 CDT 2009', 'i686'), 'sanity_size': 65536, 'type': 'cern', 'pstat': (16877, 678574248L, 20L, 1, 0, 0, 512L, 1257314403, 1257314403, 1252942658), 'mode_octal': '0100016', 'mode': 33188, 'size_bytes': 477904L, 'gname': 'root', 'gid': 0, 'fullname': '/data/data2/Migration_Spool/VO6153:0000_000000000_0032300', 'inode': 675389768L, 'minor': 20}, 'version': 'v3_8 CVS $Revision: 1.963 $ encp.pyc', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/data/data2/Migration_Spool/VO6153:0000_000000000_0032300', 'status': ('ok', None), 'outfile': '/pnfs/fs/usr/Migration/ktev/migrated_from_dlt/KQ/KQRC/KQRC52/.(access)(00280000000000000841A148)', 'fc': {'uid': 0, 'external_label': 'VOV138', 'gid': 0, 'location_cookie': '0000_000000000_0005183', 'complete_crc': 633698066L, 'bfid': 'CDMS125731442300000', 'sanity_cookie': (65536L, 1073862877L), 'size': 477904L}, 'infilepath': '/data/data2/Migration_Spool/VO6153:0000_000000000_0032300', 'at_the_top': 42, 'local_inode': 4495347L, 'resend': {'max_retry': 3, 'resubmits': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.13.19', 57442), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'encp_start_time': 1257314403.944859, 'seek_time': 0.026864051818847656, 'transfer_time': 6.1927139759063721, 't0': 1257314403, 'mount_time': 0.01845097541809082, 'lm_dequeued': 1257314417.392499, 'job_queued': 1257314404.256093, 'drive_call_time': 0.016437292098999023, 'in_queue': 13.136147975921631, 'drive_transfer_time': 6.023181676864624}, 'user_level': 'admin', 'unique_id': 'stkendm7a.fnal.gov-1257314404-18643-24632'} Thread tape_thread 00:00:23 cdfenmvr192a.fnal.gov 006151 root I LTO4_92MV Done is sent Thread tape_thread 00:00:23 cdfenmvr192a.fnal.gov 006151 root I LTO4_92MV maybe_clean Thread tape_thread 00:00:23 stkendm7a.fnal.gov 018643 root I ENCP Received final dialog for stkendm7a.fnal.gov-1257314404-18643-24632. elapsed=154670.120sec Thread WRITE_2 00:00:23 stkendm7a.fnal.gov 018643 root I ENCP leaving transfer_file(). Thread WRITE_2 00:00:23 cdfenmvr192a.fnal.gov 006151 root I LTO4_92MV returned from maybe_clean Thread tape_thread 00:00:23 stkensrv0n.fnal.gov 028796 enstore I FILSRV Updating update file set deleted = 'n', pnfs_path = '/pnfs/fs/usr/ktev/migrated_from_dlt/KQ/KQRC/KQRC52/f5012', drive = 'cdfenmvr192a:/dev/rmt/tps0d0n:1310060800', pnfs_id = '00280000000000000841A148' where bfid = 'CDMS125731442300000'; 00:00:23 cmsstor233.fnal.gov 004201 root I ENCP Creating zero length output files. 00:00:23 cmsstor233.fnal.gov 004201 root I ENCP Sending /pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0086/4E4BDD96-EEC8-DE11-A2B0-0018F3D096A0.root write request to LM: unique_id: cmsstor233.fnal.gov-1257314423-4201-0 inputfile: /storage/data1/write-pool-1/data/00070000000000006253FB80 outputfile: /pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0086/.(access)(00070000000000006253FB80) 00:00:23 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM MSG_TYPE=ADD_TO_LMQ request added to pending queue for storage group : cms 00:00:23 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM write Q'd /storage/data1/write-pool-1/data/00070000000000006253FB80 (cmsstor233.fnal.gov-1257314423-4201-0) -> /pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0086/4E4BDD96-EEC8-DE11-A2B0-0018F3D096A0.root : library=CD-LTO4G1 family=cms11 requester:root volume_family:cms.cms11.cpio_odc 00:00:23 cmsstor233.fnal.gov 004201 root I ENCP Listening for control socket at: ('131.225.191.105', 55416) 00:00:23 cmsstor233.fnal.gov 004201 root I ENCP Waiting for mover to connect control socket at address: ('131.225.191.105', 55416) elapsed=1.010sec 00:00:24 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM 00:00:24 stkendm7a.fnal.gov 018643 root I ENCP MSG_TYPE=ENCP_XFER write_to_hsm /data/data2/Migration_Spool/VO6153:0000_000000000_0032300 -> /pnfs/fs/usr/Migration/ktev/migrated_from_dlt/KQ/KQRC/KQRC52/f5012: 477904 bytes copied to VOV138 at 0.0735 MB/S (85.2 MB/S network) (0.0757 MB/S drive) (88 MB/S disk) mover=LTO4_92.mover drive_id=ULTRIUM-TD4 drive_sn=1310060800 drive_vendor=IBM elapsed=20.177 {'storage_group': 'ktev', 'encp_crc': 633698066L, 'disk_rate': 88.002466323456886, 'drive_rate': 0.075668441524589958, 'driver': 'FTTDriver', 'transfer_rate': 0.073524902640705006, 'mover_interface': 'cdfenmvr192a', 'overall_rate': 0.06805769603770527, 'media_changer': 'SL8500F1.media_changer', 'network_rate': 0.073524902640705006, 'encp_ip': '131.225.13.19', 'unique_id': 'stkendm7a.fnal.gov-1257314404-18643-24632'} Thread WRITE_2 00:00:24 stkendm7a.fnal.gov 018643 root I ENCP File /pnfs/fs/usr/Migration/ktev/migrated_from_dlt/KQ/KQRC/KQRC52/f5012 copied successfully. Thread WRITE_2 00:00:24 stkendm7a.fnal.gov 018643 root I ENCP Completed transferring 477904 bytes in 1 files in 20.1789219379 sec. Overall rate = 0.0681 MB/sec. Transfer rate = 0.0735 MB/sec. Network rate = 85.2 MB/sec. Drive rate = 0.0757 MB/sec. Disk rate = 88 MB/sec. Exit status = 0. Thread WRITE_2 00:00:24 stkensrv0n.fnal.gov 028796 enstore I FILSRV start modifying {'pnfsid': '00280000000000000841A148', 'pnfs_name0': '/pnfs/fs/usr/ktev/migrated_from_dlt/KQ/KQRC/KQRC52/f5012', 'deleted': 'no', 'external_label': 'VOV138', 'drive': 'cdfenmvr192a:/dev/rmt/tps0d0n:1310060800', 'update': '2009-11-04 00:00:23.811762', 'gid': 0, 'location_cookie': '0000_000000000_0005183', 'complete_crc': 633698066L, 'uid': 0, 'bfid': 'CDMS125731442300000', 'sanity_cookie': (65536L, 1073862877L), 'size': 477904L} 00:00:24 stkensrv0n.fnal.gov 028796 enstore I FILSRV Updating update file set pnfs_id = '001F00000000000001000C78' where bfid = 'CDMS125731442300000'; 00:00:24 stkensrv0n.fnal.gov 028796 enstore I FILSRV modified to {'pnfsid': '001F00000000000001000C78', 'pnfs_name0': '/pnfs/fs/usr/ktev/migrated_from_dlt/KQ/KQRC/KQRC52/f5012', 'deleted': 'no', 'external_label': 'VOV138', 'drive': 'cdfenmvr192a:/dev/rmt/tps0d0n:1310060800', 'update': '2009-11-04 00:00:23.811762', 'gid': 0, 'location_cookie': '0000_000000000_0005183', 'complete_crc': 633698066L, 'uid': 0, 'bfid': 'CDMS125731442300000', 'sanity_cookie': (65536L, 1073862877L), 'size': 477904L} 00:00:24 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM 00:00:24 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM write_to_hsm next work on vol=VOV138 mover=LTO4_92.mover requester:root 00:00:24 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM HAVE_BOUND:sending write_to_hsm /pnfs/fs/usr/ktev/migrated_from_dlt/KQ/KQRC/KQRC44/f6813 to mover LTO4_92.mover ('131.225.215.76', 7853) DEL_DISM 2 00:00:24 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM MSG_TYPE=ADD_TO_LMQ request added to work at movers queue for storage group : ktev 00:00:24 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM HAVE_BOUND:sending {'vc': {'storage_group': 'ktev', 'external_label': 'VOV138', 'library': 'CD-LTO4F1', 'file_family': 'ktev_migration-MIGRATION', 'wrapper': 'cern', 'address': ('131.225.13.59', 7502), 'volume_family': 'ktev.ktev_migration-MIGRATION.cern', 'file_family_width': 1}, 'outfilepath': '/pnfs/fs/usr/Migration/ktev/migrated_from_dlt/KQ/KQRC/KQRC44/f6813', 'encp': {'agetime': 30, 'adminpri': 6, 'delpri': 1, 'basepri': 1000007, 'delayed_dismount': 2, 'curpri': 1000007}, 'file_size': 121592L, 'mover': 'LTO4_92.mover', 'ignore_fair_share': 1, 'retry': None, 'lm': {'address': ('131.225.13.4', 8542)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fs/usr/ktev/migrated_from_dlt/KQ/KQRC/KQRC44/f6813', 'uid': 0, 'uname': 'root', 'type': 'cern', 'gid': 0, 'mtime': 1257314410, 'rmajor': 0, 'machine': ('Linux', 'stkendm7a.fnal.gov', '2.6.9-89.0.9.ELsmp', '#1 SMP Mon Aug 24 13:47:43 CDT 2009', 'i686'), 'sanity_size': 65536, 'pstat': (16877, 678574824L, 20L, 1, 0, 0, 512L, 1257314341, 1257314341, 1252942696), 'mode_octal': '0100016', 'mode': 33188, 'gname': 'root', 'size_bytes': 121592L, 'fullname': '/data/data2/Migration_Spool/VO6153:0000_000000000_0031889', 'inode': 675389792L, 'minor': 20}, 'version': 'v3_8 CVS $Revision: 1.963 $ encp.pyc', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/data/data2/Migration_Spool/VO6153:0000_000000000_0031889', 'status': ('ok', None), 'outfile': '/pnfs/fs/usr/Migration/ktev/migrated_from_dlt/KQ/KQRC/KQRC44/.(access)(00280000000000000841A160)', 'fc': {'size': 121592L, 'external_label': 'VOV138', 'pnfsid': '00280000000000000841A160', 'address': ('131.225.13.59', 7501)}, 'infilepath': '/data/data2/Migration_Spool/VO6153:0000_000000000_0031889', 'at_the_top': 42, 'local_inode': 33308835L, 'resend': {'max_retry': 3, 'resubmits': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.13.19', 57444), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'lm_dequeued': 1257314424.3975599, 'encp_start_time': 1257314410.213377, 'job_queued': 1257314410.5168271, 't0': 1257314410, 'in_queue': 13.880478858947754}, 'user_level': 'admin', 'unique_id': 'stkendm7a.fnal.gov-1257314410-18643-24633'} to mover 00:00:24 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM 00:00:24 cdfenmvr192a.fnal.gov 006151 root I LTO4_92MV WRITE_TO_HSM Thread MainThread 00:00:24 stkendm7a.fnal.gov 018643 root I ENCP Listening for control socket at: ('131.225.13.19', 57444) Thread WRITE 00:00:24 cdfenmvr192a.fnal.gov 006151 root I LTO4_92MV SENDING {'vc': {'storage_group': 'ktev', 'external_label': 'VOV138', 'library': 'CD-LTO4F1', 'file_family': 'ktev_migration-MIGRATION', 'wrapper': 'cern', 'address': ('131.225.13.59', 7502), 'volume_family': 'ktev.ktev_migration-MIGRATION.cern', 'file_family_width': 1}, 'outfilepath': '/pnfs/fs/usr/Migration/ktev/migrated_from_dlt/KQ/KQRC/KQRC44/f6813', 'encp': {'agetime': 30, 'adminpri': 6, 'delpri': 1, 'basepri': 1000007, 'delayed_dismount': 2, 'curpri': 1000007}, 'file_size': 121592L, 'mover': {'max_buffer': 2621440000L, 'check_written_file': 600, 'max_failures': 5, 'library': 'CD-LTO4F1.library_manager', 'local_mover': 0, 'single_filemark': 1, 'max_consecutive_failures': 3, 'norestart': 'INQ', 'port': 7853, 'mc_device': '1,0,1,1', 'compression': 0, 'serial_num': '1310060800', 'update_interval_in_bound': 2, 'update_interval': 5, 'syslog_entry': 'sense|st[0-9]', 'status': ('ok', None), 'max_rate': 73400320.0, 'vendor_id': 'IBM', 'driver': 'FTTDriver', 'host': 'cdfenmvr192a', 'max_time_in_state': 1200, 'device': 'cdfenmvr192a:/dev/rmt/tps0d0n', 'name': 'LTO4_92.mover', 'send_stats': 1, 'product_id': 'ULTRIUM-TD4', 'statistics_path': '/tmp/enstore/enstore/LTO4_92.stat', 'mover_address': ('131.225.215.76', 7853), 'mount_delay': 15, 'check_first_written_file': 0, 'callback_addr': ('131.225.215.76', 60658), 'data_ip': 'cdfenmvr192a', 'logname': 'LTO4_92MV', 'hostip': '131.225.215.76', 'do_cleaning': 'No', 'media_changer': 'SL8500F1.media_changer'}, 'ignore_fair_share': 1, 'retry': None, 'lm': {'address': ('131.225.13.4', 8542)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fs/usr/ktev/migrated_from_dlt/KQ/KQRC/KQRC44/f6813', 'uid': 0, 'uname': 'root', 'mtime': 1257314410, 'rmajor': 0, 'machine': ('Linux', 'stkendm7a.fnal.gov', '2.6.9-89.0.9.ELsmp', '#1 SMP Mon Aug 24 13:47:43 CDT 2009', 'i686'), 'sanity_size': 65536, 'type': 'cern', 'pstat': (16877, 678574824L, 20L, 1, 0, 0, 512L, 1257314341, 1257314341, 1252942696), 'mode_octal': '0100016', 'mode': 33188, 'size_bytes': 121592L, 'gname': 'root', 'gid': 0, 'fullname': '/data/data2/Migration_Spool/VO6153:0000_000000000_0031889', 'inode': 675389792L, 'minor': 20}, 'version': 'v3_8 CVS $Revision: 1.963 $ encp.pyc', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/data/data2/Migration_Spool/VO6153:0000_000000000_0031889', 'status': ('ok', None), 'outfile': '/pnfs/fs/usr/Migration/ktev/migrated_from_dlt/KQ/KQRC/KQRC44/.(access)(00280000000000000841A160)', 'fc': {'size': 121592L, 'external_label': 'VOV138', 'pnfsid': '00280000000000000841A160', 'address': ('131.225.13.59', 7501)}, 'infilepath': '/data/data2/Migration_Spool/VO6153:0000_000000000_0031889', 'at_the_top': 42, 'local_inode': 33308835L, 'resend': {'max_retry': 3, 'resubmits': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.13.19', 57444), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'lm_dequeued': 1257314424.3975599, 'encp_start_time': 1257314410.213377, 'job_queued': 1257314410.5168271, 't0': 1257314410, 'in_queue': 13.880478858947754}, 'user_level': 'admin', 'unique_id': 'stkendm7a.fnal.gov-1257314410-18643-24633'} Thread client_connect_thread 00:00:24 stkendm7a.fnal.gov 018643 root I ENCP Waiting for mover to connect control socket at address: ('131.225.13.19', 57444) elapsed=154670.860sec Thread WRITE 00:00:24 stkendm7a.fnal.gov 018643 root I ENCP Control socket ('131.225.13.19', 57444) is connected to ('131.225.215.76', 60659) for stkendm7a.fnal.gov-1257314410-18643-24633. elapsed=154670.860sec Thread WRITE 00:00:24 stkendm7a.fnal.gov 018643 root I ENCP Received callback ticket from mover LTO4_92.mover for transfer stkendm7a.fnal.gov-1257314410-18643-24633. elapsed=154670.860sec Thread WRITE 00:00:24 stkendm7a.fnal.gov 018643 root I ENCP Attempting to connect data socket to mover at ('131.225.215.76', 60658). Thread WRITE 00:00:24 stkendm7a.fnal.gov 018643 root I ENCP Connecting data socket to mover (('131.225.215.76', 60658)) with interface 131.225.13.19. elapsed=154670.870sec Thread WRITE 00:00:24 stkendm7a.fnal.gov 018643 root I ENCP Data socket ('131.225.13.19', 57449) is connected to ('131.225.215.76', 60658). elapsed=154670.870sec Thread WRITE 00:00:24 stkendm7a.fnal.gov 018643 root I ENCP Opening local file. Thread WRITE 00:00:24 stkendm7a.fnal.gov 018643 root I ENCP pf_status = ('ok', None) skip_layer_cleanup = False Thread WRITE 00:00:24 cdfenmvr192a.fnal.gov 006151 root I LTO4_92MV tape position: target 5185 current 5185 0 Thread seek_thread 00:00:24 cdfenmvr192a.fnal.gov 006151 root I LTO4_92MV write_tape starting, bytes_to_write=121856 Thread tape_thread 00:00:24 cdfenmvr192a.fnal.gov 006151 root I LTO4_92MV Write starting. Tape VOV138 absolute location in blocks 255464 Thread tape_thread 00:00:24 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:24 stkendm7a.fnal.gov 018643 root I ENCP File /data/data2/Migration_Spool/VO6153:0000_000000000_0031889 transfered for stkendm7a.fnal.gov-1257314410-18643-24633 in 0.0237350463867 seconds. Thread WRITE 00:00:24 stkendm7a.fnal.gov 018643 root I ENCP Version: v3_8 CVS $Revision: 1.963 $ encp.pyc OS: Linux 2.6.9-89.0.9.ELsmp i686 Release: Scientific Linux Fermi LTS release 4.4 (Wilson) User: root(0) Group: root(0) Euser: root(0) Egroup: root(0) Library: CD-LTO4F1 Storage Group: ktev File Family: ktev_migration-MIGRATION FF Wrapper: cern FF Width: 1 Current working directory: stkendm7a.fnal.gov:/opt/enstore/src Command line: encp --delayed-dismount 2 --ignore-fair-share --threaded --priority 0 --storage-group ktev --file-family ktev_migration-MIGRATION --file-family-wrapper cern --library CD-LTO4F1 --override-path /pnfs/fs/usr/ktev/migrated_from_dlt/KQ/KQRC/KQRC44/f1969 --threaded /data/data2/Migration_Spool/VO6153:0000_000000000_0032309 /pnfs/fs/usr/Migration/ktev/migrated_from_dlt/KQ/KQRC/KQRC44/f1969 Thread WRITE_2 00:00:24 stkendm7a.fnal.gov 018643 root I ENCP Creating zero length output files. Thread WRITE_2 00:00:24 stkendm7a.fnal.gov 018643 root I ENCP Sending /pnfs/fs/usr/Migration/ktev/migrated_from_dlt/KQ/KQRC/KQRC44/f1969 write request to LM: unique_id: stkendm7a.fnal.gov-1257314424-18643-24635 inputfile: /data/data2/Migration_Spool/VO6153:0000_000000000_0032309 outputfile: /pnfs/fs/usr/Migration/ktev/migrated_from_dlt/KQ/KQRC/KQRC44/.(access)(00280000000000000841A198) Thread WRITE_2 00:00:24 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM MSG_TYPE=ADD_TO_LMQ request added to pending queue for storage group : ktev 00:00:24 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM write Q'd /data/data2/Migration_Spool/VO6153:0000_000000000_0032309 (stkendm7a.fnal.gov-1257314424-18643-24635) -> /pnfs/fs/usr/ktev/migrated_from_dlt/KQ/KQRC/KQRC44/f1969 : library=CD-LTO4F1 family=ktev_migration-MIGRATION requester:root volume_family:ktev.ktev_migration-MIGRATION.cern 00:00:24 stkenmvr144a.fnal.gov 006329 root I LTO4_44MV filemarks written. Tape VOR980 absolute location in blocks 1771061 Thread tape_thread 00:00:24 stkenmvr144a.fnal.gov 006329 root I LTO4_44MV write_tape timing:pull 0.42529797554 write 16.8318068981 crc_check 16.4557769299 freespace 0.421053171158 block_write 34.1339349747 idle 0.666555643082 Thread tape_thread 00:00:24 stkenmvr144a.fnal.gov 006329 root I LTO4_44MV drive stats after write. Tape VOR980 position 69 block 0 block_size 0 bloc_loc 1771061 tot_blocks 762939000 BOT 0 read_err 0 write_err 0 bytes 4115627520 block_write_tot 34.1339349747 tape_rate 120572899.757 Thread tape_thread 00:00:24 stkenmvr144a.fnal.gov 006329 root I LTO4_44MV remaining bytes info in DB 567730176000 estimated 563614548480 reported from drive 563913728000 position 70 Thread tape_thread 00:00:24 stkenmvr144a.fnal.gov 006329 root I LTO4_44MV new bitfile request {'uid': 0, 'external_label': 'VOR980', 'gid': 0, 'location_cookie': '0000_000000000_0000069', 'complete_crc': 1516227194L, 'sanity_cookie': (65536L, 2557765325L), 'size': 4115626884L} Thread tape_thread 00:00:25 stkenmvr151a.fnal.gov 013228 root I LTO4_51MV filemarks written. Tape VOQ829 absolute location in blocks 3908066 Thread tape_thread 00:00:25 stkenmvr151a.fnal.gov 013228 root I LTO4_51MV write_tape timing:pull 0.676644802094 write 20.2208342552 crc_check 23.7116932869 freespace 0.61869096756 block_write 45.2278633118 idle 0.996133565903 Thread tape_thread 00:00:25 stkenmvr151a.fnal.gov 013228 root I LTO4_51MV drive stats after write. Tape VOQ829 position 113 block 0 block_size 0 bloc_loc 3908066 tot_blocks 762939000 BOT 0 read_err 0 write_err 0 bytes 5912685568 block_write_tot 45.2278633118 tape_rate 130731039.122 Thread tape_thread 00:00:25 stkenmvr151a.fnal.gov 013228 root I LTO4_51MV remaining bytes info in DB 309217280000 estimated 303304594432 reported from drive 302055424000 position 114 Thread tape_thread 00:00:25 stkenmvr151a.fnal.gov 013228 root I LTO4_51MV new bitfile request {'uid': 0, 'external_label': 'VOQ829', 'gid': 0, 'location_cookie': '0000_000000000_0000113', 'complete_crc': 2205477614L, 'sanity_cookie': (65536L, 2707305450L), 'size': 5912684891L} Thread tape_thread 00:00:25 stkenmvr144a.fnal.gov 006329 root I LTO4_44MV set remaining: VOR980 563913728000 0000_000000000_0000070 Thread tape_thread 00:00:25 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:25 last message repeated 1 times 00:00:25 stkensrv0n.fnal.gov 028762 enstore I VOLSRV Updating update volume set last_access = '2009-11-04 00:00:25', eod_cookie = '0000_000000000_0000070', non_del_files = '69', remaining_bytes = 563913728000 where label = 'VOR980'; 00:00:25 stkenmvr151a.fnal.gov 013228 root I LTO4_51MV set remaining: VOQ829 302055424000 0000_000000000_0000114 Thread tape_thread 00:00:25 last message repeated 1 times 00:00:25 stkensrv0n.fnal.gov 028762 enstore I VOLSRV Updating update volume set last_access = '2009-11-04 00:00:25', eod_cookie = '0000_000000000_0000114', non_del_files = '113', remaining_bytes = 302055424000 where label = 'VOQ829'; 00:00:25 stkenmvr144a.fnal.gov 006329 root I LTO4_44MV transfer complete volume=VOR980 location=70 Thread tape_thread 00:00:25 last message repeated 1 times 00:00:25 stkensrv0n.fnal.gov 028762 enstore I VOLSRV Updating update volume set sum_wr_access = '69', modification_time = '2009-11-04 00:00:25' where label = 'VOR980'; 00:00:25 stkenmvr144a.fnal.gov 006329 root I LTO4_44MV Sending done to client: {'vc': {'comment': '', 'storage_group': 'cms', 'blocksize': 131072, 'declared': 1255359339.0, 'library': 'CD-LTO4G1', 'si_time': [0.0, 1250991351.0], 'user_inhibit': ['none', 'none'], 'system_inhibit': ['none', 'none'], 'external_label': 'VOR980', 'wrapper': 'cpio_odc', 'remaining_bytes': 567730176000L, 'sum_mounts': 66, 'sum_rd_access': 0, 'file_family_width': 10, 'last_access': 1257314380.0, 'media_type': 'LTO4', 'status': ('ok', None), 'eod_cookie': '0000_000000000_0000069', 'capacity_bytes': 858993459200L, 'non_del_files': 68, 'sum_wr_err': 0, 'sum_wr_access': 68, 'file_family': 'cms11', 'address': ('131.225.13.59', 7502), 'volume_family': 'cms.cms11.cpio_odc', 'modification_time': 1257314380.0, 'write_protected': 'n', 'sum_rd_err': 0, 'first_access': 1257311277.0}, 'outfilepath': '/pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0079/3CB2CD61-ACC8-DE11-9EF7-002618943923.root', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 25, 'delayed_dismount': None, 'curpri': 25}, 'file_size': 4115626884L, 'mover': {'max_buffer': 2621440000L, 'check_written_file': 600, 'max_failures': 5, 'library': 'CD-LTO4G1.library_manager', 'local_mover': 0, 'dismount_delay': 30, 'max_consecutive_failures': 3, 'norestart': 'INQ', 'port': 7844, 'mc_device': '0,6,1,15', 'compression': 0, 'serial_num': '9310123734', 'update_interval_in_bound': 2, 'update_interval': 5, 'syslog_entry': 'sense|st[0-9]', 'status': ('ok', None), 'max_rate': 73400320.0, 'vendor_id': 'IBM', 'driver': 'FTTDriver', 'host': 'stkenmvr144a', 'max_time_in_state': 1200, 'device': 'stkenmvr144a:/dev/rmt/tps0d0n', 'name': 'LTO4_44.mover', 'send_stats': 1, 'product_id': 'ULTRIUM-TD4', 'statistics_path': '/tmp/enstore/enstore/LTO4_44.stat', 'single_filemark': 1, 'mount_delay': 15, 'mover_address': ('131.225.13.172', 7844), 'check_first_written_file': 0, 'callback_addr': ('131.225.13.172', 52289), 'data_ip': 'stkenmvr144a', 'logname': 'LTO4_44MV', 'hostip': '131.225.13.172', 'do_cleaning': 'No', 'media_changer': 'SL8500G1.media_changer', 'min_buffer': 67108864L}, 'ignore_fair_share': None, 'retry': None, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0079/3CB2CD61-ACC8-DE11-9EF7-002618943923.root', 'uid': 0, 'uname': 'root', 'mtime': 1257312754, 'rmajor': 0, 'machine': ('Linux', 'cmsstor82.fnal.gov', '2.6.31', '#1 SMP Mon Sep 14 09:43:20 CDT 2009', 'x86_64'), 'sanity_size': 65536, 'type': 'cpio_odc', 'pstat': (16893, 119680632, 21L, 1, 9811, 5063, 512, 1257283077, 1257283077, 1257275646), 'mode_octal': '0100036', 'mode': 33188, 'size_bytes': 4115626884L, 'gname': 'root', 'gid': 0, 'fullname': '/storage/data3/write-pool-3/data/000700000000000062228078', 'inode': 119701624L, 'minor': 21}, 'version': 'v3_7d CVS $Revision: 1.922 $ encp', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/storage/data3/write-pool-3/data/000700000000000062228078', 'status': ('ok', None), 'outfile': '/pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0079/.(access)(000700000000000062228078)', 'fc': {'uid': 0, 'external_label': 'VOR980', 'gid': 0, 'location_cookie': '0000_000000000_0000069', 'complete_crc': 1516227194L, 'bfid': 'CDMS125731442400000', 'sanity_cookie': (65536L, 2557765325L), 'size': 4115626884L}, 'infilepath': '/storage/data3/write-pool-3/data/000700000000000062228078', 'at_the_top': 32, 'local_inode': 2147486348L, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.189.82', 46274), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'encp_start_time': 1257312753.7273281, 'seek_time': 0.035422801971435547, 'transfer_time': 43.58744215965271, 't0': 1257312753, 'mount_time': 0.041280031204223633, 'lm_dequeued': 1257314381.5380261, 'job_queued': 1257312755.0587161, 'drive_call_time': 16.392260074615479, 'in_queue': 1626.478982925415, 'drive_transfer_time': 36.733075857162476}, 'user_level': 'admin', 'unique_id':(trunc. 4121) Thread tape_thread 00:00:25 stkenmvr144a.fnal.gov 006329 root I LTO4_44MV Done is sent Thread tape_thread 00:00:25 stkenmvr144a.fnal.gov 006329 root I LTO4_44MV maybe_clean Thread tape_thread 00:00:25 stkenmvr144a.fnal.gov 006329 root I LTO4_44MV returned from maybe_clean Thread tape_thread 00:00:25 cmsstor82.fnal.gov 014013 root I ENCP Received final dialog for cmsstor82.fnal.gov-1257312754-14013-0. elapsed=1671.710sec 00:00:25 cmsstor82.fnal.gov 014013 root I ENCP leaving transfer_file(). 00:00:25 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM 00:00:25 stkenmvr151a.fnal.gov 013228 root I LTO4_51MV transfer complete volume=VOQ829 location=114 Thread tape_thread 00:00:25 last message repeated 1 times 00:00:25 stkensrv0n.fnal.gov 028762 enstore I VOLSRV Updating update volume set sum_wr_access = '113', modification_time = '2009-11-04 00:00:25' where label = 'VOQ829'; 00:00:25 cmsstor133.fnal.gov 013052 root I ENCP File /storage/data2/write-pool-2/data/00070000000000006257AFA0 transfered for cmsstor133.fnal.gov-1257312791-13052-0 in 23.9491689205 seconds. 00:00:25 stkenmvr151a.fnal.gov 013228 root I LTO4_51MV Sending done to client: {'vc': {'comment': '', 'storage_group': 'cms', 'blocksize': 131072, 'declared': 1255358452.0, 'library': 'CD-LTO4G1', 'si_time': [0.0, 1241498402.0], 'user_inhibit': ['none', 'none'], 'system_inhibit': ['none', 'none'], 'external_label': 'VOQ829', 'wrapper': 'cpio_odc', 'remaining_bytes': 309217280000L, 'sum_mounts': 118, 'sum_rd_access': 0, 'file_family_width': 15, 'last_access': 1257313593.0, 'media_type': 'LTO4', 'status': ('ok', None), 'eod_cookie': '0000_000000000_0000113', 'capacity_bytes': 858993459200L, 'non_del_files': 112, 'sum_wr_err': 0, 'sum_wr_access': 112, 'file_family': 'NonCustodial', 'address': ('131.225.13.59', 7502), 'volume_family': 'cms.NonCustodial.cpio_odc', 'modification_time': 1257313593.0, 'write_protected': 'n', 'sum_rd_err': 0, 'first_access': 1256992087.0}, 'outfilepath': '/pnfs/fnal.gov/usr/cms/WAX//11/store/data/Commissioning09/Cosmics/RAW/v3/000/119/406/74955A60-ECC8-DE11-9261-000423D992DC.root', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 25, 'delayed_dismount': None, 'curpri': 25}, 'file_size': 5912684891L, 'mover': {'max_buffer': 2621440000L, 'check_written_file': 600, 'max_failures': 5, 'library': 'CD-LTO4G1.library_manager', 'local_mover': 0, 'dismount_delay': 30, 'max_consecutive_failures': 3, 'norestart': 'INQ', 'port': 7851, 'mc_device': '0,5,1,12', 'compression': 0, 'serial_num': '1310154445', 'update_interval_in_bound': 2, 'update_interval': 5, 'syslog_entry': 'sense|st[0-9]', 'status': ('ok', None), 'max_rate': 73400320.0, 'vendor_id': 'IBM', 'driver': 'FTTDriver', 'host': 'stkenmvr151a', 'max_time_in_state': 1200, 'device': 'stkenmvr151a:/dev/rmt/tps0d0n', 'name': 'LTO4_51.mover', 'send_stats': 1, 'product_id': 'ULTRIUM-TD4', 'statistics_path': '/tmp/enstore/enstore/LTO4_51.stat', 'single_filemark': 1, 'mount_delay': 15, 'mover_address': ('131.225.13.179', 7851), 'check_first_written_file': 0, 'callback_addr': ('131.225.13.179', 38127), 'data_ip': 'stkenmvr151a', 'logname': 'LTO4_51MV', 'hostip': '131.225.13.179', 'do_cleaning': 'No', 'media_changer': 'SL8500G1.media_changer', 'min_buffer': 67108864L}, 'ignore_fair_share': None, 'retry': None, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fnal.gov/usr/cms/WAX//11/store/data/Commissioning09/Cosmics/RAW/v3/000/119/406/74955A60-ECC8-DE11-9261-000423D992DC.root', 'uid': 0, 'uname': 'root', 'mtime': 1257314142, 'rmajor': 0, 'machine': ('Linux', 'cmsstor79.fnal.gov', '2.6.31', '#1 SMP Mon Sep 14 09:43:20 CDT 2009', 'x86_64'), 'sanity_size': 65536, 'type': 'cpio_odc', 'pstat': (16877, 123003976, 21L, 1, 9811, 5063, 512, 1257314107, 1257314107, 1257305045), 'mode_octal': '0100016', 'mode': 33188, 'size_bytes': 5912684891L, 'gname': 'root', 'gid': 0, 'fullname': '/storage/data2/write-pool-2/data/000700000000000062604C70', 'inode': 123751536L, 'minor': 21}, 'version': 'v3_7d CVS $Revision: 1.922 $ encp', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/storage/data2/write-pool-2/data/000700000000000062604C70', 'status': ('ok', None), 'outfile': '/pnfs/fnal.gov/usr/cms/WAX//11/store/data/Commissioning09/Cosmics/RAW/v3/000/119/406/.(access)(000700000000000062604C70)', 'fc': {'uid': 0, 'external_label': 'VOQ829', 'gid': 0, 'location_cookie': '0000_000000000_0000113', 'complete_crc': 2205477614L, 'bfid': 'CDMS125731442500000', 'sanity_cookie': (65536L, 2707305450L), 'size': 5912684891L}, 'infilepath': '/storage/data2/write-pool-2/data/000700000000000062604C70', 'at_the_top': 160, 'local_inode': 1073745791, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.205.201', 57389), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'encp_start_time': 1257314141.3205259, 'seek_time': 155.56515789031982, 'transfer_time': 83.531695127487183, 't0': 1257314141, 'lm_dequeued': 1257314186.2313349, 'job_queued': 1257314142.739996, 'drive_call_time': 19.604905605316162, 'in_queue': 43.491276025772095, 'drive_transfer_time': 48.312633752822876}, 'user_level': 'admin', '(trunc. 4131) Thread tape_thread 00:00:25 stkenmvr151a.fnal.gov 013228 root I LTO4_51MV Done is sent Thread tape_thread 00:00:25 stkenmvr151a.fnal.gov 013228 root I LTO4_51MV maybe_clean Thread tape_thread 00:00:25 stkenmvr151a.fnal.gov 013228 root I LTO4_51MV returned from maybe_clean Thread tape_thread 00:00:25 cmsstor79.fnal.gov 006780 root I ENCP Received final dialog for cmsstor79.fnal.gov-1257314142-6780-0. elapsed=284.130sec 00:00:25 cmsstor79.fnal.gov 006780 root I ENCP leaving transfer_file(). 00:00:25 cmsstor82.fnal.gov 014013 root I ENCP pf_status = ('ok', None) skip_layer_cleanup = False 00:00:25 cdfenmvr192a.fnal.gov 006151 root I LTO4_92MV written bytes 121856/121856, blocks 1 header 0 trailer 264 Thread tape_thread 00:00:25 cmsstor79.fnal.gov 006780 root I ENCP pf_status = ('ok', None) skip_layer_cleanup = False 00:00:25 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:25 last message repeated 1 times 00:00:25 stkensrv0n.fnal.gov 028796 enstore I FILSRV Updating update file set deleted = 'n', pnfs_path = '/pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0079/3CB2CD61-ACC8-DE11-9EF7-002618943923.root', drive = 'stkenmvr144a:/dev/rmt/tps0d0n:9310123734', pnfs_id = '000700000000000062228078' where bfid = 'CDMS125731442400000'; 00:00:25 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM write_to_hsm next work on vol=VOR980 mover=LTO4_44.mover requester:root 00:00:25 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM HAVE_BOUND:sending write_to_hsm /pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0084/AC98CF3B-D9C8-DE11-AE0E-002618FDA207.root to mover LTO4_44.mover ('131.225.13.172', 7844) DEL_DISM None 00:00:25 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM MSG_TYPE=ADD_TO_LMQ request added to work at movers queue for storage group : cms 00:00:25 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM HAVE_BOUND:sending {'vc': {'storage_group': 'cms', 'external_label': 'VOR980', 'library': 'CD-LTO4G1', 'file_family': 'cms11', 'wrapper': 'cpio_odc', 'address': ('131.225.13.59', 7502), 'volume_family': 'cms.cms11.cpio_odc', 'file_family_width': 10}, 'outfilepath': '/pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0084/AC98CF3B-D9C8-DE11-AE0E-002618FDA207.root', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 25, 'delayed_dismount': None, 'curpri': 25}, 'file_size': 3095657456L, 'mover': 'LTO4_44.mover', 'ignore_fair_share': None, 'retry': None, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0084/AC98CF3B-D9C8-DE11-AE0E-002618FDA207.root', 'uid': 0, 'uname': 'root', 'type': 'cpio_odc', 'gid': 0, 'mtime': 1257312803, 'rmajor': 0, 'machine': ('Linux', 'cmsstor95.fnal.gov', '2.6.31', '#1 SMP Mon Sep 14 09:43:20 CDT 2009', 'x86_64'), 'sanity_size': 65536, 'pstat': (16893, 121659864, 23L, 1, 9811, 5063, 512, 1257299007, 1257299007, 1257292342), 'mode_octal': '0100036', 'mode': 33188, 'gname': 'root', 'size_bytes': 3095657456L, 'fullname': '/storage/data3/write-pool-3/data/00070000000000006245AB98', 'inode': 122006424L, 'minor': 23}, 'version': 'v3_7d CVS $Revision: 1.922 $ encp', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/storage/data3/write-pool-3/data/00070000000000006245AB98', 'status': ('ok', None), 'outfile': '/pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0084/.(access)(00070000000000006245AB98)', 'fc': {'size': 3095657456L, 'external_label': 'VOR980', 'address': ('131.225.13.59', 7501)}, 'infilepath': '/storage/data3/write-pool-3/data/00070000000000006245AB98', 'at_the_top': 16, 'local_inode': 1073744412, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.205.217', 37006), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'lm_dequeued': 1257314425.5814011, 'encp_start_time': 1257312802.300772, 'job_queued': 1257312803.7747469, 't0': 1257312802, 'in_queue': 1621.8063361644745}, 'user_level': 'admin', 'unique_id': 'cmsstor95.fnal.gov-1257312803-10572-0'} to mover 00:00:25 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:25 stkenmvr144a.fnal.gov 006329 root I LTO4_44MV WRITE_TO_HSM Thread MainThread 00:00:25 stkenmvr144a.fnal.gov 006329 root I LTO4_44MV SENDING {'vc': {'storage_group': 'cms', 'external_label': 'VOR980', 'library': 'CD-LTO4G1', 'file_family': 'cms11', 'wrapper': 'cpio_odc', 'address': ('131.225.13.59', 7502), 'volume_family': 'cms.cms11.cpio_odc', 'file_family_width': 10}, 'outfilepath': '/pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0084/AC98CF3B-D9C8-DE11-AE0E-002618FDA207.root', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 25, 'delayed_dismount': None, 'curpri': 25}, 'file_size': 3095657456L, 'mover': {'max_buffer': 2621440000L, 'check_written_file': 600, 'max_failures': 5, 'library': 'CD-LTO4G1.library_manager', 'local_mover': 0, 'dismount_delay': 30, 'max_consecutive_failures': 3, 'norestart': 'INQ', 'port': 7844, 'mc_device': '0,6,1,15', 'compression': 0, 'serial_num': '9310123734', 'update_interval_in_bound': 2, 'update_interval': 5, 'syslog_entry': 'sense|st[0-9]', 'status': ('ok', None), 'max_rate': 73400320.0, 'vendor_id': 'IBM', 'driver': 'FTTDriver', 'host': 'stkenmvr144a', 'max_time_in_state': 1200, 'device': 'stkenmvr144a:/dev/rmt/tps0d0n', 'name': 'LTO4_44.mover', 'send_stats': 1, 'product_id': 'ULTRIUM-TD4', 'statistics_path': '/tmp/enstore/enstore/LTO4_44.stat', 'single_filemark': 1, 'mount_delay': 15, 'mover_address': ('131.225.13.172', 7844), 'check_first_written_file': 0, 'callback_addr': ('131.225.13.172', 52296), 'data_ip': 'stkenmvr144a', 'logname': 'LTO4_44MV', 'hostip': '131.225.13.172', 'do_cleaning': 'No', 'media_changer': 'SL8500G1.media_changer', 'min_buffer': 67108864L}, 'ignore_fair_share': None, 'retry': None, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0084/AC98CF3B-D9C8-DE11-AE0E-002618FDA207.root', 'uid': 0, 'uname': 'root', 'mtime': 1257312803, 'rmajor': 0, 'machine': ('Linux', 'cmsstor95.fnal.gov', '2.6.31', '#1 SMP Mon Sep 14 09:43:20 CDT 2009', 'x86_64'), 'sanity_size': 65536, 'type': 'cpio_odc', 'pstat': (16893, 121659864, 23L, 1, 9811, 5063, 512, 1257299007, 1257299007, 1257292342), 'mode_octal': '0100036', 'mode': 33188, 'size_bytes': 3095657456L, 'gname': 'root', 'gid': 0, 'fullname': '/storage/data3/write-pool-3/data/00070000000000006245AB98', 'inode': 122006424L, 'minor': 23}, 'version': 'v3_7d CVS $Revision: 1.922 $ encp', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/storage/data3/write-pool-3/data/00070000000000006245AB98', 'status': ('ok', None), 'outfile': '/pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0084/.(access)(00070000000000006245AB98)', 'fc': {'size': 3095657456L, 'external_label': 'VOR980', 'address': ('131.225.13.59', 7501)}, 'infilepath': '/storage/data3/write-pool-3/data/00070000000000006245AB98', 'at_the_top': 16, 'local_inode': 1073744412, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.205.217', 37006), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'lm_dequeued': 1257314425.5814011, 'encp_start_time': 1257312802.300772, 'job_queued': 1257312803.7747469, 't0': 1257312802, 'in_queue': 1621.8063361644745}, 'user_level': 'admin', 'unique_id': 'cmsstor95.fnal.gov-1257312803-10572-0'} Thread client_connect_thread 00:00:25 cmsstor95.fnal.gov 010572 root I ENCP Control socket ('131.225.205.217', 37006) is connected to ('131.225.13.172', 52297) for cmsstor95.fnal.gov-1257312803-10572-0. elapsed=1623.470sec 00:00:25 cmsstor95.fnal.gov 010572 root I ENCP Received callback ticket from mover LTO4_44.mover for transfer cmsstor95.fnal.gov-1257312803-10572-0. elapsed=1623.470sec 00:00:25 cmsstor95.fnal.gov 010572 root I ENCP Atempting to connect data socket to mover at ('131.225.13.172', 52296). 00:00:25 cmsstor95.fnal.gov 010572 root I ENCP Connecting data socket to mover (('131.225.13.172', 52296)) with interface 131.225.205.217. elapsed=1623.480sec 00:00:25 cmsstor95.fnal.gov 010572 root I ENCP Data socket ('131.225.205.217', 36725) is connected to ('131.225.13.172', 52296). elapsed=1623.480sec 00:00:25 stkensrv0n.fnal.gov 028796 enstore I FILSRV Updating update file set deleted = 'n', pnfs_path = '/pnfs/fnal.gov/usr/cms/WAX//11/store/data/Commissioning09/Cosmics/RAW/v3/000/119/406/74955A60-ECC8-DE11-9261-000423D992DC.root', drive = 'stkenmvr151a:/dev/rmt/tps0d0n:1310154445', pnfs_id = '000700000000000062604C70' where bfid = 'CDMS125731442500000'; 00:00:25 cmsstor81.fnal.gov 025587 root I ENCP Version: v3_7d CVS $Revision: 1.922 $ encp OS: Linux 2.6.31 x86_64 Release: Rocks release 5.1 (V.I) Scientific Linux SL release 5.2 (Boron) User: root(0) Group: root(0) Euser: root(0) Egroup: root(0) Library: CD-LTO4G1 Storage Group: cms File Family: cms11 FF Wrapper: cpio_odc FF Width: 10 Current working directory: cmsstor81.fnal.gov:/ Command line: /home/enstore/enstore/bin/encp --verbose 4 --threaded --ecrc --resubmit-timeout 1800 --priority 25 --pnfs-mount /pnfs/fs --put-cache 0007000000000000622A7448 /storage/data4/write-pool-4/data/0007000000000000622A7448 00:00:25 stkenmvr144a.fnal.gov 006329 root I LTO4_44MV tape position: target 70 current 70 0 Thread seek_thread 00:00:25 stkenmvr144a.fnal.gov 006329 root I LTO4_44MV write_tape starting, bytes_to_write=3095657984 Thread tape_thread 00:00:25 stkenmvr144a.fnal.gov 006329 root I LTO4_44MV Write starting. Tape VOR980 absolute location in blocks 1771061 Thread tape_thread 00:00:25 cmsstor82.fnal.gov 014013 root I ENCP MSG_TYPE=ENCP_XFER write_to_hsm /storage/data3/write-pool-3/data/000700000000000062228078 -> /pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0079/3CB2CD61-ACC8-DE11-9EF7-002618943923.root: 4115626884 bytes copied to VOR980 at 90 MB/S (89.4 MB/S network) (107 MB/S drive) (103 MB/S disk) mover=LTO4_44.mover drive_id=ULTRIUM-TD4 drive_sn=9310123734 drive_vendor=IBM elapsed=1672.1 {'storage_group': 'cms', 'encp_crc': 1516227194L, 'disk_rate': 102.76972497991426, 'drive_rate': 106.85104809747553, 'driver': 'FTTDriver', 'transfer_rate': 90.011028859734296, 'mover_interface': 'stkenmvr144a', 'overall_rate': 88.873844314614701, 'media_changer': 'SL8500G1.media_changer', 'network_rate': 90.011028859734296, 'encp_ip': '131.225.189.82', 'unique_id': 'cmsstor82.fnal.gov-1257312754-14013-0'} 00:00:25 cmsstor82.fnal.gov 014013 root I ENCP File /pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0079/3CB2CD61-ACC8-DE11-9EF7-002618943923.root copied successfully. 00:00:25 cmsstor82.fnal.gov 014013 root I ENCP Completed transferring 4115626884 bytes in 1 files in 1672.0574789 sec. Overall rate = 88.9 MB/sec. Transfer rate = 90 MB/sec. Network rate = 89.4 MB/sec. Drive rate = 107 MB/sec. Disk rate = 103 MB/sec. Exit status = 0. 00:00:25 cmsstor95.fnal.gov 010572 root I ENCP Opening local file. 00:00:25 cmsstor79.fnal.gov 006780 root I ENCP MSG_TYPE=ENCP_XFER write_to_hsm /storage/data2/write-pool-2/data/000700000000000062604C70 -> /pnfs/fnal.gov/usr/cms/WAX//11/store/data/Commissioning09/Cosmics/RAW/v3/000/119/406/74955A60-ECC8-DE11-9261-000423D992DC.root: 5912684891 bytes copied to VOQ829 at 67.5 MB/S (97.2 MB/S network) (117 MB/S drive) (70.6 MB/S disk) mover=LTO4_51.mover drive_id=ULTRIUM-TD4 drive_sn=1310154445 drive_vendor=IBM elapsed=284.56 {'storage_group': 'cms', 'encp_crc': 2205477614L, 'disk_rate': 70.632629493574569, 'drive_rate': 116.71431018228266, 'driver': 'FTTDriver', 'transfer_rate': 67.494490521398703, 'mover_interface': 'stkenmvr151a', 'overall_rate': 23.535138540828509, 'media_changer': 'SL8500G1.media_changer', 'network_rate': 67.494490521398703, 'encp_ip': '131.225.205.201', 'unique_id': 'cmsstor79.fnal.gov-1257314142-6780-0'} 00:00:25 cmsstor79.fnal.gov 006780 root I ENCP File /pnfs/fnal.gov/usr/cms/WAX//11/store/data/Commissioning09/Cosmics/RAW/v3/000/119/406/74955A60-ECC8-DE11-9261-000423D992DC.root copied successfully. 00:00:25 cmsstor79.fnal.gov 006780 root I ENCP Completed transferring 5912684891 bytes in 1 files in 284.559575081 sec. Overall rate = 23.5 MB/sec. Transfer rate = 67.5 MB/sec. Network rate = 97.2 MB/sec. Drive rate = 117 MB/sec. Disk rate = 70.6 MB/sec. Exit status = 0. 00:00:26 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM 00:00:26 cmsstor95.fnal.gov 010572 root I ENCP pf_status = ('ok', None) skip_layer_cleanup = False 00:00:26 stkenmvr149a.fnal.gov 006318 root I LTO4_49MV written bytes 2816669184/2816669184, blocks 21490 header 192 trailer 522 Thread tape_thread 00:00:26 cmsstor81.fnal.gov 025587 root I ENCP Creating zero length output files. 00:00:26 cmsstor81.fnal.gov 025587 root I ENCP Sending /pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0080/D4E00D89-B5C8-DE11-9DAE-0018F3D09650.root write request to LM: unique_id: cmsstor81.fnal.gov-1257314426-25587-0 inputfile: /storage/data4/write-pool-4/data/0007000000000000622A7448 outputfile: /pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0080/.(access)(0007000000000000622A7448) 00:00:26 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM MSG_TYPE=ADD_TO_LMQ request added to pending queue for storage group : cms 00:00:26 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM write Q'd /storage/data4/write-pool-4/data/0007000000000000622A7448 (cmsstor81.fnal.gov-1257314426-25587-0) -> /pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0080/D4E00D89-B5C8-DE11-9DAE-0018F3D09650.root : library=CD-LTO4G1 family=cms11 requester:root volume_family:cms.cms11.cpio_odc 00:00:26 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:26 cmsstor81.fnal.gov 025587 root I ENCP Listening for control socket at: ('131.225.205.203', 57476) 00:00:26 cmsstor81.fnal.gov 025587 root I ENCP Waiting for mover to connect control socket at address: ('131.225.205.203', 57476) elapsed=1.290sec 00:00:26 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:26 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:26 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:26 stkenmvr205a.fnal.gov 011191 root I LTO4_105MV written bytes 2268467712/2268467712, blocks 17308 header 192 trailer 267 Thread tape_thread 00:00:27 stkenmvr149a.fnal.gov 006318 root I LTO4_49MV filemarks written. Tape VOR958 absolute location in blocks 2762871 Thread tape_thread 00:00:27 stkenmvr149a.fnal.gov 006318 root I LTO4_49MV write_tape timing:pull 0.255725860596 write 10.2891871929 crc_check 11.135835886 freespace 0.274492025375 block_write 21.9552409649 idle 0.432334899902 Thread tape_thread 00:00:27 stkenmvr149a.fnal.gov 006318 root I LTO4_49MV drive stats after write. Tape VOR958 position 113 block 0 block_size 0 bloc_loc 2762871 tot_blocks 762939000 BOT 0 read_err 0 write_err 0 bytes 2816669184 block_write_tot 21.9552409649 tape_rate 128291426.567 Thread tape_thread 00:00:27 stkenmvr149a.fnal.gov 006318 root I LTO4_49MV remaining bytes info in DB 441508864000 estimated 438692194816 reported from drive 438857728000 position 114 Thread tape_thread 00:00:27 stkenmvr149a.fnal.gov 006318 root I LTO4_49MV new bitfile request {'uid': 0, 'external_label': 'VOR958', 'gid': 0, 'location_cookie': '0000_000000000_0000113', 'complete_crc': 3771266389L, 'sanity_cookie': (65536L, 2177570226L), 'size': 2816668470L} Thread tape_thread 00:00:27 cmsstor206.fnal.gov 001751 root I ENCP File /storage/data2/write-pool-2/data/0007000000000000622F1F78 transfered for cmsstor206.fnal.gov-1257312799-1751-0 in 22.2201559544 seconds. 00:00:27 stkenmvr149a.fnal.gov 006318 root I LTO4_49MV set remaining: VOR958 438857728000 0000_000000000_0000114 Thread tape_thread 00:00:27 last message repeated 1 times 00:00:27 stkensrv0n.fnal.gov 028762 enstore I VOLSRV Updating update volume set last_access = '2009-11-04 00:00:27', eod_cookie = '0000_000000000_0000114', non_del_files = '113', remaining_bytes = 438857728000 where label = 'VOR958'; 00:00:27 stkenmvr149a.fnal.gov 006318 root I LTO4_49MV transfer complete volume=VOR958 location=114 Thread tape_thread 00:00:27 last message repeated 1 times 00:00:27 stkensrv0n.fnal.gov 028762 enstore I VOLSRV Updating update volume set sum_wr_access = '113', modification_time = '2009-11-04 00:00:27' where label = 'VOR958'; 00:00:27 stkenmvr149a.fnal.gov 006318 root I LTO4_49MV Sending done to client: {'vc': {'comment': '', 'storage_group': 'cms', 'blocksize': 131072, 'declared': 1255359329.0, 'library': 'CD-LTO4G1', 'si_time': [0.0, 1250979729.0], 'user_inhibit': ['none', 'none'], 'system_inhibit': ['none', 'none'], 'external_label': 'VOR958', 'wrapper': 'cpio_odc', 'remaining_bytes': 441508864000L, 'sum_mounts': 63, 'sum_rd_access': 0, 'file_family_width': 10, 'last_access': 1257314400.0, 'media_type': 'LTO4', 'status': ('ok', None), 'eod_cookie': '0000_000000000_0000113', 'capacity_bytes': 858993459200L, 'non_del_files': 112, 'sum_wr_err': 0, 'sum_wr_access': 112, 'file_family': 'cms11', 'address': ('131.225.13.59', 7502), 'volume_family': 'cms.cms11.cpio_odc', 'modification_time': 1257314400.0, 'write_protected': 'n', 'sum_rd_err': 0, 'first_access': 1257309371.0}, 'outfilepath': '/pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0088/90EF8EB4-F2C8-DE11-80CF-001731AF65EF.root', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 25, 'delayed_dismount': None, 'curpri': 25}, 'file_size': 2816668470L, 'mover': {'max_buffer': 2621440000L, 'check_written_file': 50, 'max_failures': 5, 'library': 'CD-LTO4G1.library_manager', 'local_mover': 0, 'dismount_delay': 30, 'max_consecutive_failures': 3, 'norestart': 'INQ', 'port': 7849, 'mc_device': '0,5,1,3', 'compression': 0, 'serial_num': '1310154355', 'update_interval_in_bound': 2, 'update_interval': 5, 'syslog_entry': 'sense|st[0-9]', 'status': ('ok', None), 'max_rate': 73400320.0, 'vendor_id': 'IBM', 'driver': 'FTTDriver', 'host': 'stkenmvr149a', 'max_time_in_state': 1200, 'device': 'stkenmvr149a:/dev/rmt/tps0d0n', 'name': 'LTO4_49.mover', 'send_stats': 1, 'product_id': 'ULTRIUM-TD4', 'statistics_path': '/tmp/enstore/enstore/LTO4_49.stat', 'single_filemark': 1, 'mount_delay': 15, 'mover_address': ('131.225.13.177', 7849), 'check_first_written_file': 0, 'callback_addr': ('131.225.13.177', 41204), 'data_ip': 'stkenmvr149a', 'logname': 'LTO4_49MV', 'hostip': '131.225.13.177', 'do_cleaning': 'No', 'media_changer': 'SL8500G1.media_changer', 'min_buffer': 67108864L}, 'ignore_fair_share': None, 'retry': None, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0088/90EF8EB4-F2C8-DE11-80CF-001731AF65EF.root', 'uid': 0, 'uname': 'root', 'mtime': 1257312791, 'rmajor': 0, 'machine': ('Linux', 'cmsstor133.fnal.gov', '2.6.31', '#1 SMP Mon Sep 14 09:43:20 CDT 2009', 'x86_64'), 'sanity_size': 65536, 'type': 'cpio_odc', 'pstat': (16893, 123093320, 21L, 1, 9811, 5063, 512, 1257312531, 1257312531, 1257305807), 'mode_octal': '0100036', 'mode': 33188, 'size_bytes': 2816668470L, 'gname': 'root', 'gid': 0, 'fullname': '/storage/data2/write-pool-2/data/00070000000000006257AFA0', 'inode': 123187104L, 'minor': 21}, 'version': 'v3_7d CVS $Revision: 1.922 $ encp', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/storage/data2/write-pool-2/data/00070000000000006257AFA0', 'status': ('ok', None), 'outfile': '/pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0088/.(access)(00070000000000006257AFA0)', 'fc': {'uid': 0, 'external_label': 'VOR958', 'gid': 0, 'location_cookie': '0000_000000000_0000113', 'complete_crc': 3771266389L, 'bfid': 'CDMS125731442700000', 'sanity_cookie': (65536L, 2177570226L), 'size': 2816668470L}, 'infilepath': '/storage/data2/write-pool-2/data/00070000000000006257AFA0', 'at_the_top': 20, 'local_inode': 851, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.189.104', 58099), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'encp_start_time': 1257312791.0142181, 'seek_time': 0.034384965896606445, 'transfer_time': 26.37833309173584, 't0': 1257312791, 'mount_time': 0.0445098876953125, 'lm_dequeued': 1257314401.294903, 'job_queued': 1257312791.9767179, 'drive_call_time': 9.9842991828918457, 'in_queue': 1609.3175859451294, 'drive_transfer_time': 24.221370458602905}, 'user_level': 'admin', 'unique_id': 'cmssto(trunc. 4114) Thread tape_thread 00:00:27 stkenmvr149a.fnal.gov 006318 root I LTO4_49MV Done is sent Thread tape_thread 00:00:27 stkenmvr149a.fnal.gov 006318 root I LTO4_49MV maybe_clean Thread tape_thread 00:00:27 cmsstor133.fnal.gov 013052 root I ENCP Received final dialog for cmsstor133.fnal.gov-1257312791-13052-0. elapsed=1636.650sec 00:00:27 cmsstor133.fnal.gov 013052 root I ENCP leaving transfer_file(). 00:00:27 stkenmvr149a.fnal.gov 006318 root I LTO4_49MV returned from maybe_clean Thread tape_thread 00:00:27 cmsstor133.fnal.gov 013052 root I ENCP pf_status = ('ok', None) skip_layer_cleanup = False 00:00:28 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM 00:00:28 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:28 stkensrv0n.fnal.gov 028796 enstore I FILSRV Updating update file set deleted = 'n', pnfs_path = '/pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0088/90EF8EB4-F2C8-DE11-80CF-001731AF65EF.root', drive = 'stkenmvr149a:/dev/rmt/tps0d0n:1310154355', pnfs_id = '00070000000000006257AFA0' where bfid = 'CDMS125731442700000'; 00:00:28 cmsstor82.fnal.gov 013130 root I ENCP Version: v3_7d CVS $Revision: 1.922 $ encp OS: Linux 2.6.31 x86_64 Release: Rocks release 5.1 (V.I) Scientific Linux SL release 5.2 (Boron) User: root(0) Group: root(0) Euser: root(0) Egroup: root(0) Library: CD-LTO4G1 Storage Group: cms File Family: cms11 FF Wrapper: cpio_odc FF Width: 10 Current working directory: cmsstor82.fnal.gov:/ Command line: /home/enstore/enstore/bin/encp --verbose 4 --threaded --ecrc --resubmit-timeout 1800 --priority 25 --pnfs-mount /pnfs/fs --put-cache 00070000000000006225DCC8 /storage/data2/write-pool-2/data/00070000000000006225DCC8 00:00:28 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:28 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM write_to_hsm next work on vol=VOR958 mover=LTO4_49.mover requester:root 00:00:28 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM HAVE_BOUND:sending write_to_hsm /pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0085/D4523C79-DEC8-DE11-937B-0026189438B9.root to mover LTO4_49.mover ('131.225.13.177', 7849) DEL_DISM None 00:00:28 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM MSG_TYPE=ADD_TO_LMQ request added to work at movers queue for storage group : cms 00:00:28 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM HAVE_BOUND:sending {'vc': {'storage_group': 'cms', 'external_label': 'VOR958', 'library': 'CD-LTO4G1', 'file_family': 'cms11', 'wrapper': 'cpio_odc', 'address': ('131.225.13.59', 7502), 'volume_family': 'cms.cms11.cpio_odc', 'file_family_width': 10}, 'outfilepath': '/pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0085/D4523C79-DEC8-DE11-937B-0026189438B9.root', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 25, 'delayed_dismount': None, 'curpri': 25}, 'file_size': 1826707529L, 'mover': 'LTO4_49.mover', 'ignore_fair_share': None, 'retry': None, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0085/D4523C79-DEC8-DE11-937B-0026189438B9.root', 'uid': 0, 'uname': 'root', 'type': 'cpio_odc', 'gid': 0, 'mtime': 1257312812, 'rmajor': 0, 'machine': ('Linux', 'cmsstor225.fnal.gov', '2.6.31', '#1 SMP Mon Sep 14 09:43:20 CDT 2009', 'x86_64'), 'sanity_size': 65536, 'pstat': (16893, 122155992, 21L, 1, 9811, 5063, 512, 1257304578, 1257304578, 1257296604), 'mode_octal': '0100036', 'mode': 33188, 'gname': 'root', 'size_bytes': 1826707529L, 'fullname': '/storage/data2/write-pool-2/data/00070000000000006248A680', 'inode': 122201728L, 'minor': 21}, 'version': 'v3_7d CVS $Revision: 1.922 $ encp', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/storage/data2/write-pool-2/data/00070000000000006248A680', 'status': ('ok', None), 'outfile': '/pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0085/.(access)(00070000000000006248A680)', 'fc': {'size': 1826707529L, 'external_label': 'VOR958', 'address': ('131.225.13.59', 7501)}, 'infilepath': '/storage/data2/write-pool-2/data/00070000000000006248A680', 'at_the_top': 12, 'local_inode': 6776, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.191.97', 53605), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'lm_dequeued': 1257314428.3179481, 'encp_start_time': 1257312812.0100811, 'job_queued': 1257312812.8667729, 't0': 1257312812, 'in_queue': 1615.4506380558014}, 'user_level': 'admin', 'unique_id': 'cmsstor225.fnal.gov-1257312812-8281-0'} to mover 00:00:28 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:28 stkenmvr149a.fnal.gov 006318 root I LTO4_49MV WRITE_TO_HSM Thread MainThread 00:00:28 cmsstor133.fnal.gov 013052 root I ENCP MSG_TYPE=ENCP_XFER write_to_hsm /storage/data2/write-pool-2/data/00070000000000006257AFA0 -> /pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0088/90EF8EB4-F2C8-DE11-80CF-001731AF65EF.root: 2816668470 bytes copied to VOR958 at 102 MB/S (94.6 MB/S network) (111 MB/S drive) (112 MB/S disk) mover=LTO4_49.mover drive_id=ULTRIUM-TD4 drive_sn=1310154355 drive_vendor=IBM elapsed=1637.3 {'storage_group': 'cms', 'encp_crc': 3771266389L, 'disk_rate': 112.18648640155277, 'drive_rate': 110.90141997791928, 'driver': 'FTTDriver', 'transfer_rate': 101.80674458528223, 'mover_interface': 'stkenmvr149a', 'overall_rate': 99.475024543460037, 'media_changer': 'SL8500G1.media_changer', 'network_rate': 101.80674458528223, 'encp_ip': '131.225.189.104', 'unique_id': 'cmsstor133.fnal.gov-1257312791-13052-0'} 00:00:28 stkenmvr149a.fnal.gov 006318 root I LTO4_49MV SENDING {'vc': {'storage_group': 'cms', 'external_label': 'VOR958', 'library': 'CD-LTO4G1', 'file_family': 'cms11', 'wrapper': 'cpio_odc', 'address': ('131.225.13.59', 7502), 'volume_family': 'cms.cms11.cpio_odc', 'file_family_width': 10}, 'outfilepath': '/pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0085/D4523C79-DEC8-DE11-937B-0026189438B9.root', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 25, 'delayed_dismount': None, 'curpri': 25}, 'file_size': 1826707529L, 'mover': {'max_buffer': 2621440000L, 'check_written_file': 50, 'max_failures': 5, 'library': 'CD-LTO4G1.library_manager', 'local_mover': 0, 'dismount_delay': 30, 'max_consecutive_failures': 3, 'norestart': 'INQ', 'port': 7849, 'mc_device': '0,5,1,3', 'compression': 0, 'serial_num': '1310154355', 'update_interval_in_bound': 2, 'update_interval': 5, 'syslog_entry': 'sense|st[0-9]', 'status': ('ok', None), 'max_rate': 73400320.0, 'vendor_id': 'IBM', 'driver': 'FTTDriver', 'host': 'stkenmvr149a', 'max_time_in_state': 1200, 'device': 'stkenmvr149a:/dev/rmt/tps0d0n', 'name': 'LTO4_49.mover', 'send_stats': 1, 'product_id': 'ULTRIUM-TD4', 'statistics_path': '/tmp/enstore/enstore/LTO4_49.stat', 'single_filemark': 1, 'mount_delay': 15, 'mover_address': ('131.225.13.177', 7849), 'check_first_written_file': 0, 'callback_addr': ('131.225.13.177', 41210), 'data_ip': 'stkenmvr149a', 'logname': 'LTO4_49MV', 'hostip': '131.225.13.177', 'do_cleaning': 'No', 'media_changer': 'SL8500G1.media_changer', 'min_buffer': 67108864L}, 'ignore_fair_share': None, 'retry': None, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0085/D4523C79-DEC8-DE11-937B-0026189438B9.root', 'uid': 0, 'uname': 'root', 'mtime': 1257312812, 'rmajor': 0, 'machine': ('Linux', 'cmsstor225.fnal.gov', '2.6.31', '#1 SMP Mon Sep 14 09:43:20 CDT 2009', 'x86_64'), 'sanity_size': 65536, 'type': 'cpio_odc', 'pstat': (16893, 122155992, 21L, 1, 9811, 5063, 512, 1257304578, 1257304578, 1257296604), 'mode_octal': '0100036', 'mode': 33188, 'size_bytes': 1826707529L, 'gname': 'root', 'gid': 0, 'fullname': '/storage/data2/write-pool-2/data/00070000000000006248A680', 'inode': 122201728L, 'minor': 21}, 'version': 'v3_7d CVS $Revision: 1.922 $ encp', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/storage/data2/write-pool-2/data/00070000000000006248A680', 'status': ('ok', None), 'outfile': '/pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0085/.(access)(00070000000000006248A680)', 'fc': {'size': 1826707529L, 'external_label': 'VOR958', 'address': ('131.225.13.59', 7501)}, 'infilepath': '/storage/data2/write-pool-2/data/00070000000000006248A680', 'at_the_top': 12, 'local_inode': 6776, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.191.97', 53605), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'lm_dequeued': 1257314428.3179481, 'encp_start_time': 1257312812.0100811, 'job_queued': 1257312812.8667729, 't0': 1257312812, 'in_queue': 1615.4506380558014}, 'user_level': 'admin', 'unique_id': 'cmsstor225.fnal.gov-1257312812-8281-0'} Thread client_connect_thread 00:00:28 cmsstor133.fnal.gov 013052 root I ENCP File /pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0088/90EF8EB4-F2C8-DE11-80CF-001731AF65EF.root copied successfully. 00:00:28 cmsstor133.fnal.gov 013052 root I ENCP Completed transferring 2816668470 bytes in 1 files in 1637.32091093 sec. Overall rate = 99.5 MB/sec. Transfer rate = 102 MB/sec. Network rate = 94.6 MB/sec. Drive rate = 111 MB/sec. Disk rate = 112 MB/sec. Exit status = 0. 00:00:28 cmsstor225.fnal.gov 008281 root I ENCP Control socket ('131.225.191.97', 53605) is connected to ('131.225.13.177', 41211) for cmsstor225.fnal.gov-1257312812-8281-0. elapsed=1616.350sec 00:00:28 cmsstor225.fnal.gov 008281 root I ENCP Received callback ticket from mover LTO4_49.mover for transfer cmsstor225.fnal.gov-1257312812-8281-0. elapsed=1616.360sec 00:00:28 cmsstor225.fnal.gov 008281 root I ENCP Atempting to connect data socket to mover at ('131.225.13.177', 41210). 00:00:28 cmsstor225.fnal.gov 008281 root I ENCP Connecting data socket to mover (('131.225.13.177', 41210)) with interface 131.225.191.97. elapsed=1616.360sec 00:00:28 cmsstor225.fnal.gov 008281 root I ENCP Data socket ('131.225.191.97', 48038) is connected to ('131.225.13.177', 41210). elapsed=1616.360sec 00:00:28 cmsstor225.fnal.gov 008281 root I ENCP Opening local file. 00:00:28 stkenmvr205a.fnal.gov 011191 root I LTO4_105MV filemarks written. Tape VOS309 absolute location in blocks 719952 Thread tape_thread 00:00:28 stkenmvr205a.fnal.gov 011191 root I LTO4_105MV write_tape timing:pull 0.264836072922 write 6.98821783066 crc_check 10.4864654541 freespace 0.181675195694 block_write 17.9211945534 idle 0.313418626785 Thread tape_thread 00:00:28 stkenmvr205a.fnal.gov 011191 root I LTO4_105MV drive stats after write. Tape VOS309 position 29 block 0 block_size 0 bloc_loc 719952 tot_blocks 762939000 BOT 0 read_err 0 write_err 0 bytes 2268467712 block_write_tot 17.9211945534 tape_rate 126580162.123 Thread tape_thread 00:00:28 stkenmvr205a.fnal.gov 011191 root I LTO4_105MV remaining bytes info in DB 692170752000 estimated 689902284288 reported from drive 690028544000 position 30 Thread tape_thread 00:00:28 stkenmvr205a.fnal.gov 011191 root I LTO4_105MV new bitfile request {'uid': 0, 'external_label': 'VOS309', 'gid': 0, 'location_cookie': '0000_000000000_0000029', 'complete_crc': 2838776578L, 'sanity_cookie': (65536L, 1246388102L), 'size': 2268467253L} Thread tape_thread 00:00:28 stkenmvr149a.fnal.gov 006318 root I LTO4_49MV tape position: target 114 current 114 0 Thread seek_thread 00:00:28 cmsstor225.fnal.gov 008281 root I ENCP pf_status = ('ok', None) skip_layer_cleanup = False 00:00:28 stkenmvr149a.fnal.gov 006318 root I LTO4_49MV write_tape starting, bytes_to_write=1826707968 Thread tape_thread 00:00:28 stkenmvr205a.fnal.gov 011191 root I LTO4_105MV set remaining: VOS309 690028544000 0000_000000000_0000030 Thread tape_thread 00:00:28 last message repeated 1 times 00:00:28 stkenmvr149a.fnal.gov 006318 root I LTO4_49MV Write starting. Tape VOR958 absolute location in blocks 2762871 Thread tape_thread 00:00:28 stkensrv0n.fnal.gov 028762 enstore I VOLSRV Updating update volume set last_access = '2009-11-04 00:00:28', eod_cookie = '0000_000000000_0000030', non_del_files = '29', remaining_bytes = 690028544000 where label = 'VOS309'; 00:00:28 stkenmvr208a.fnal.gov 006111 root I LTO4_108MV skip_fm 268 Thread seek_thread 00:00:28 stkenmvr208a.fnal.gov 006111 root I LTO4_108MV write_tape starting, bytes_to_write=1122435584 Thread tape_thread 00:00:28 stkenmvr208a.fnal.gov 006111 root I LTO4_108MV Write starting. Tape VON730 absolute location in blocks 3985894 Thread tape_thread 00:00:28 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:28 cmsstor82.fnal.gov 013130 root I ENCP Creating zero length output files. 00:00:28 cmsstor82.fnal.gov 013130 root I ENCP Sending /pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0079/9A8696CB-B0C8-DE11-A39C-00261894397F.root write request to LM: unique_id: cmsstor82.fnal.gov-1257314428-13130-0 inputfile: /storage/data2/write-pool-2/data/00070000000000006225DCC8 outputfile: /pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0079/.(access)(00070000000000006225DCC8) 00:00:28 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM MSG_TYPE=ADD_TO_LMQ request added to pending queue for storage group : cms 00:00:28 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM write Q'd /storage/data2/write-pool-2/data/00070000000000006225DCC8 (cmsstor82.fnal.gov-1257314428-13130-0) -> /pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0079/9A8696CB-B0C8-DE11-A39C-00261894397F.root : library=CD-LTO4G1 family=cms11 requester:root volume_family:cms.cms11.cpio_odc 00:00:28 stkenmvr205a.fnal.gov 011191 root I LTO4_105MV transfer complete volume=VOS309 location=30 Thread tape_thread 00:00:28 cmsstor82.fnal.gov 013130 root I ENCP Listening for control socket at: ('131.225.189.82', 56633) 00:00:28 cmsstor82.fnal.gov 013130 root I ENCP Waiting for mover to connect control socket at address: ('131.225.189.82', 56633) elapsed=1.250sec 00:00:28 last message repeated 1 times 00:00:28 stkensrv0n.fnal.gov 028762 enstore I VOLSRV Updating update volume set sum_wr_access = '29', modification_time = '2009-11-04 00:00:28' where label = 'VOS309'; 00:00:28 stkenmvr205a.fnal.gov 011191 root I LTO4_105MV Sending done to client: {'vc': {'comment': '', 'storage_group': 'cms', 'blocksize': 131072, 'declared': 1255359565.0, 'library': 'CD-LTO4G1', 'si_time': [0.0, 1250990162.0], 'user_inhibit': ['none', 'none'], 'system_inhibit': ['none', 'none'], 'external_label': 'VOS309', 'wrapper': 'cpio_odc', 'remaining_bytes': 692170752000L, 'sum_mounts': 50, 'sum_rd_access': 0, 'file_family_width': 10, 'last_access': 1257314392.0, 'media_type': 'LTO4', 'status': ('ok', None), 'eod_cookie': '0000_000000000_0000029', 'capacity_bytes': 858993459200L, 'non_del_files': 28, 'sum_wr_err': 0, 'sum_wr_access': 28, 'file_family': 'cms11', 'address': ('131.225.13.59', 7502), 'volume_family': 'cms.cms11.cpio_odc', 'modification_time': 1257314392.0, 'write_protected': 'n', 'sum_rd_err': 0, 'first_access': 1257313061.0}, 'outfilepath': '/pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0083/5AF75A97-D1C8-DE11-BB15-0030486791BA.root', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 25, 'delayed_dismount': None, 'curpri': 25}, 'file_size': 2268467253L, 'mover': {'max_buffer': 2621440000L, 'check_written_file': 600, 'max_failures': 5, 'library': 'CD-LTO4G1.library_manager', 'local_mover': 0, 'single_filemark': 1, 'max_consecutive_failures': 3, 'norestart': 'INQ', 'port': 7865, 'mc_device': '0,6,1,1', 'compression': 0, 'serial_num': '9310801322', 'update_interval_in_bound': 2, 'update_interval': 5, 'syslog_entry': 'sense|st[0-9]', 'status': ('ok', None), 'max_rate': 73400320.0, 'vendor_id': 'IBM', 'driver': 'FTTDriver', 'host': 'stkenmvr205a', 'max_time_in_state': 1200, 'device': 'stkenmvr205a:/dev/rmt/tps0d0n', 'name': 'LTO4_105.mover', 'send_stats': 1, 'product_id': 'ULTRIUM-TD4', 'statistics_path': '/tmp/enstore/enstore/LTO4_105.stat', 'mover_address': ('131.225.13.201', 7865), 'mount_delay': 15, 'check_first_written_file': 0, 'callback_addr': ('131.225.13.201', 37825), 'data_ip': 'stkenmvr205a', 'logname': 'LTO4_105MV', 'hostip': '131.225.13.201', 'do_cleaning': 'No', 'media_changer': 'SL8500G1.media_changer'}, 'ignore_fair_share': None, 'retry': None, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0083/5AF75A97-D1C8-DE11-BB15-0030486791BA.root', 'uid': 0, 'uname': 'root', 'mtime': 1257312771, 'rmajor': 0, 'machine': ('Linux', 'cmsstor65.fnal.gov', '2.6.31', '#1 SMP Mon Sep 14 09:43:20 CDT 2009', 'x86_64'), 'sanity_size': 65536, 'type': 'cpio_odc', 'pstat': (16893, 121380464, 21L, 1, 9811, 5063, 512, 1257294382, 1257294382, 1257289995), 'mode_octal': '0100036', 'mode': 33188, 'size_bytes': 2268467253L, 'gname': 'root', 'gid': 0, 'fullname': '/storage/data2/write-pool-2/data/0007000000000000623EF7E8', 'inode': 121567208L, 'minor': 21}, 'version': 'v3_7d CVS $Revision: 1.922 $ encp', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/storage/data2/write-pool-2/data/0007000000000000623EF7E8', 'status': ('ok', None), 'outfile': '/pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0083/.(access)(0007000000000000623EF7E8)', 'fc': {'uid': 0, 'external_label': 'VOS309', 'gid': 0, 'location_cookie': '0000_000000000_0000029', 'complete_crc': 2838776578L, 'bfid': 'CDMS125731442800000', 'sanity_cookie': (65536L, 1246388102L), 'size': 2268467253L}, 'infilepath': '/storage/data2/write-pool-2/data/0007000000000000623EF7E8', 'at_the_top': 30, 'local_inode': 1073742718, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.204.241', 60261), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'encp_start_time': 1257312770.2749009, 'seek_time': 0.029883146286010742, 'transfer_time': 35.606703042984009, 't0': 1257312770, 'mount_time': 0.014209985733032227, 'lm_dequeued': 1257314392.9748311, 'job_queued': 1257312771.4535289, 'drive_call_time': 6.751340389251709, 'in_queue': 1621.5210020542145, 'drive_transfer_time': 20.035339593887329}, 'user_level': 'admin', 'unique_id': 'cmsstor65.fnal.gov-1257312771-29700-0'} Thread tape_thread 00:00:28 stkenmvr205a.fnal.gov 011191 root I LTO4_105MV Done is sent Thread tape_thread 00:00:28 stkenmvr205a.fnal.gov 011191 root I LTO4_105MV maybe_clean Thread tape_thread 00:00:28 stkenmvr205a.fnal.gov 011191 root I LTO4_105MV returned from maybe_clean Thread tape_thread 00:00:28 cmsstor65.fnal.gov 029700 root I ENCP Received final dialog for cmsstor65.fnal.gov-1257312771-29700-0. elapsed=1658.620sec 00:00:28 cmsstor65.fnal.gov 029700 root I ENCP leaving transfer_file(). 00:00:28 stkendca22a.fnal.gov 010203 root I ENCP File /diskb/write-pool-2/data/000F0000000000000B20C180 transfered for stkendca22a.fnal.gov-1257314235-10203-0 in 19.6186339855 seconds. 00:00:28 stkenmvr197a.fnal.gov 025501 root I LTO4_97MV written bytes 1594721280/1594721280, blocks 12167 header 179 trailer 127 Thread tape_thread 00:00:28 stkenmvr206a.fnal.gov 026766 root I LTO4_106MV tape position: target 115 current 0 1 Thread seek_thread 00:00:28 cmsstor65.fnal.gov 029700 root I ENCP pf_status = ('ok', None) skip_layer_cleanup = False 00:00:28 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:29 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM write_to_hsm next work on vol=VOS309 mover=LTO4_105.mover requester:root 00:00:29 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM HAVE_BOUND:sending write_to_hsm /pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0085/BA495B73-DEC8-DE11-9211-002618FDA259.root to mover LTO4_105.mover ('131.225.13.201', 7865) DEL_DISM None 00:00:29 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM MSG_TYPE=ADD_TO_LMQ request added to work at movers queue for storage group : cms 00:00:29 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM HAVE_BOUND:sending {'vc': {'storage_group': 'cms', 'external_label': 'VOS309', 'library': 'CD-LTO4G1', 'file_family': 'cms11', 'wrapper': 'cpio_odc', 'address': ('131.225.13.59', 7502), 'volume_family': 'cms.cms11.cpio_odc', 'file_family_width': 10}, 'outfilepath': '/pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0085/BA495B73-DEC8-DE11-9211-002618FDA259.root', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 25, 'delayed_dismount': None, 'curpri': 25}, 'file_size': 2750644566L, 'mover': 'LTO4_105.mover', 'ignore_fair_share': None, 'retry': None, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0085/BA495B73-DEC8-DE11-9211-002618FDA259.root', 'uid': 0, 'uname': 'root', 'type': 'cpio_odc', 'gid': 0, 'mtime': 1257312832, 'rmajor': 0, 'machine': ('Linux', 'cmsstor187.fnal.gov', '2.6.31', '#1 SMP Mon Sep 14 09:43:20 CDT 2009', 'x86_64'), 'sanity_size': 65536, 'pstat': (16893, 122155992, 23L, 1, 9811, 5063, 512, 1257304578, 1257304578, 1257296604), 'mode_octal': '0100036', 'mode': 33188, 'gname': 'root', 'size_bytes': 2750644566L, 'fullname': '/storage/data1/write-pool-1/data/0007000000000000624912E0', 'inode': 122229472L, 'minor': 23}, 'version': 'v3_7d CVS $Revision: 1.922 $ encp', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/storage/data1/write-pool-1/data/0007000000000000624912E0', 'status': ('ok', None), 'outfile': '/pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0085/.(access)(0007000000000000624912E0)', 'fc': {'size': 2750644566L, 'external_label': 'VOS309', 'address': ('131.225.13.59', 7501)}, 'infilepath': '/storage/data1/write-pool-1/data/0007000000000000624912E0', 'at_the_top': 2, 'local_inode': 6037, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.207.10', 36296), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'lm_dequeued': 1257314429.01753, 'encp_start_time': 1257312831.707052, 'job_queued': 1257312832.998575, 't0': 1257312831, 'in_queue': 1596.0186450481415}, 'user_level': 'admin', 'unique_id': 'cmsstor187.fnal.gov-1257312832-1249-0'} to mover 00:00:29 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:29 stkenmvr205a.fnal.gov 011191 root I LTO4_105MV WRITE_TO_HSM Thread MainThread 00:00:29 stkenmvr205a.fnal.gov 011191 root I LTO4_105MV SENDING {'vc': {'storage_group': 'cms', 'external_label': 'VOS309', 'library': 'CD-LTO4G1', 'file_family': 'cms11', 'wrapper': 'cpio_odc', 'address': ('131.225.13.59', 7502), 'volume_family': 'cms.cms11.cpio_odc', 'file_family_width': 10}, 'outfilepath': '/pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0085/BA495B73-DEC8-DE11-9211-002618FDA259.root', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 25, 'delayed_dismount': None, 'curpri': 25}, 'file_size': 2750644566L, 'mover': {'max_buffer': 2621440000L, 'check_written_file': 600, 'max_failures': 5, 'library': 'CD-LTO4G1.library_manager', 'local_mover': 0, 'single_filemark': 1, 'max_consecutive_failures': 3, 'norestart': 'INQ', 'port': 7865, 'mc_device': '0,6,1,1', 'compression': 0, 'serial_num': '9310801322', 'update_interval_in_bound': 2, 'update_interval': 5, 'syslog_entry': 'sense|st[0-9]', 'status': ('ok', None), 'max_rate': 73400320.0, 'vendor_id': 'IBM', 'driver': 'FTTDriver', 'host': 'stkenmvr205a', 'max_time_in_state': 1200, 'device': 'stkenmvr205a:/dev/rmt/tps0d0n', 'name': 'LTO4_105.mover', 'send_stats': 1, 'product_id': 'ULTRIUM-TD4', 'statistics_path': '/tmp/enstore/enstore/LTO4_105.stat', 'mover_address': ('131.225.13.201', 7865), 'mount_delay': 15, 'check_first_written_file': 0, 'callback_addr': ('131.225.13.201', 37828), 'data_ip': 'stkenmvr205a', 'logname': 'LTO4_105MV', 'hostip': '131.225.13.201', 'do_cleaning': 'No', 'media_changer': 'SL8500G1.media_changer'}, 'ignore_fair_share': None, 'retry': None, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0085/BA495B73-DEC8-DE11-9211-002618FDA259.root', 'uid': 0, 'uname': 'root', 'mtime': 1257312832, 'rmajor': 0, 'machine': ('Linux', 'cmsstor187.fnal.gov', '2.6.31', '#1 SMP Mon Sep 14 09:43:20 CDT 2009', 'x86_64'), 'sanity_size': 65536, 'type': 'cpio_odc', 'pstat': (16893, 122155992, 23L, 1, 9811, 5063, 512, 1257304578, 1257304578, 1257296604), 'mode_octal': '0100036', 'mode': 33188, 'size_bytes': 2750644566L, 'gname': 'root', 'gid': 0, 'fullname': '/storage/data1/write-pool-1/data/0007000000000000624912E0', 'inode': 122229472L, 'minor': 23}, 'version': 'v3_7d CVS $Revision: 1.922 $ encp', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/storage/data1/write-pool-1/data/0007000000000000624912E0', 'status': ('ok', None), 'outfile': '/pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0085/.(access)(0007000000000000624912E0)', 'fc': {'size': 2750644566L, 'external_label': 'VOS309', 'address': ('131.225.13.59', 7501)}, 'infilepath': '/storage/data1/write-pool-1/data/0007000000000000624912E0', 'at_the_top': 2, 'local_inode': 6037, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.207.10', 36296), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'lm_dequeued': 1257314429.01753, 'encp_start_time': 1257312831.707052, 'job_queued': 1257312832.998575, 't0': 1257312831, 'in_queue': 1596.0186450481415}, 'user_level': 'admin', 'unique_id': 'cmsstor187.fnal.gov-1257312832-1249-0'} Thread client_connect_thread 00:00:29 cmsstor187.fnal.gov 001249 root I ENCP Control socket ('131.225.207.10', 36296) is connected to ('131.225.13.201', 37829) for cmsstor187.fnal.gov-1257312832-1249-0. elapsed=1597.350sec 00:00:29 stkensrv0n.fnal.gov 028796 enstore I FILSRV Updating update file set deleted = 'n', pnfs_path = '/pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0083/5AF75A97-D1C8-DE11-BB15-0030486791BA.root', drive = 'stkenmvr205a:/dev/rmt/tps0d0n:9310801322', pnfs_id = '0007000000000000623EF7E8' where bfid = 'CDMS125731442800000'; 00:00:29 cmsstor187.fnal.gov 001249 root I ENCP Received callback ticket from mover LTO4_105.mover for transfer cmsstor187.fnal.gov-1257312832-1249-0. elapsed=1597.350sec 00:00:29 cmsstor187.fnal.gov 001249 root I ENCP Atempting to connect data socket to mover at ('131.225.13.201', 37828). 00:00:29 cmsstor187.fnal.gov 001249 root I ENCP Connecting data socket to mover (('131.225.13.201', 37828)) with interface 131.225.207.10. elapsed=1597.350sec 00:00:29 cmsstor187.fnal.gov 001249 root I ENCP Data socket ('131.225.207.10', 53198) is connected to ('131.225.13.201', 37828). elapsed=1597.350sec 00:00:29 cmsstor187.fnal.gov 001249 root I ENCP Opening local file. 00:00:29 stkenmvr205a.fnal.gov 011191 root I LTO4_105MV tape position: target 30 current 30 0 Thread seek_thread 00:00:29 stkenmvr205a.fnal.gov 011191 root I LTO4_105MV write_tape starting, bytes_to_write=2750645248 Thread tape_thread 00:00:29 stkenmvr205a.fnal.gov 011191 root I LTO4_105MV Write starting. Tape VOS309 absolute location in blocks 719952 Thread tape_thread 00:00:29 cmsstor187.fnal.gov 001249 root I ENCP pf_status = ('ok', None) skip_layer_cleanup = False 00:00:29 cmsstor65.fnal.gov 029700 root I ENCP MSG_TYPE=ENCP_XFER write_to_hsm /storage/data2/write-pool-2/data/0007000000000000623EF7E8 -> /pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0083/5AF75A97-D1C8-DE11-BB15-0030486791BA.root: 2268467253 bytes copied to VOS309 at 61 MB/S (84 MB/S network) (108 MB/S drive) (85.8 MB/S disk) mover=LTO4_105.mover drive_id=ULTRIUM-TD4 drive_sn=9310801322 drive_vendor=IBM elapsed=1659 {'storage_group': 'cms', 'encp_crc': 2838776578L, 'disk_rate': 85.829580571345517, 'drive_rate': 107.97815263659288, 'driver': 'FTTDriver', 'transfer_rate': 61.030661819351501, 'mover_interface': 'stkenmvr205a', 'overall_rate': 59.7575741810045, 'media_changer': 'SL8500G1.media_changer', 'network_rate': 61.030661819351501, 'encp_ip': '131.225.204.241', 'unique_id': 'cmsstor65.fnal.gov-1257312771-29700-0'} 00:00:29 cmsstor65.fnal.gov 029700 root I ENCP File /pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0083/5AF75A97-D1C8-DE11-BB15-0030486791BA.root copied successfully. 00:00:29 cmsstor65.fnal.gov 029700 root I ENCP Completed transferring 2268467253 bytes in 1 files in 1658.98776913 sec. Overall rate = 59.8 MB/sec. Transfer rate = 61 MB/sec. Network rate = 84 MB/sec. Drive rate = 108 MB/sec. Disk rate = 85.8 MB/sec. Exit status = 0. 00:00:29 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:29 cdfenmvr192a.fnal.gov 006151 root I LTO4_92MV filemarks written. Tape VOV138 absolute location in blocks 255470 Thread tape_thread 00:00:29 cdfenmvr192a.fnal.gov 006151 root I LTO4_92MV write_tape timing:pull 7.15255737305e-06 write 7.89165496826e-05 crc_check 0.000848054885864 freespace 5.96046447754e-06 block_write 0.000940084457397 idle 1.81198120117e-05 Thread tape_thread 00:00:29 cdfenmvr192a.fnal.gov 006151 root I LTO4_92MV drive stats after write. Tape VOV138 position 5185 block 0 block_size 0 bloc_loc 255470 tot_blocks 762939000 BOT 0 read_err 0 write_err 14 bytes 121856 block_write_tot 0.000940084457397 tape_rate 129622396.202 Thread tape_thread 00:00:29 cdfenmvr192a.fnal.gov 006151 root I LTO4_92MV remaining bytes info in DB 738919424000 estimated 738919302144 reported from drive 738913280000 position 5188 Thread tape_thread 00:00:29 cdfenmvr192a.fnal.gov 006151 root I LTO4_92MV new bitfile request {'uid': 0, 'external_label': 'VOV138', 'gid': 0, 'location_cookie': '0000_000000000_0005186', 'complete_crc': 2933529775L, 'sanity_cookie': (65536L, 1732236251L), 'size': 121592L} Thread tape_thread 00:00:29 cdfenmvr192a.fnal.gov 006151 root I LTO4_92MV set remaining: VOV138 738913280000 0000_000000000_0005188 Thread tape_thread 00:00:29 last message repeated 1 times 00:00:29 stkensrv0n.fnal.gov 028762 enstore I VOLSRV Updating update volume set last_access = '2009-11-04 00:00:29', eod_cookie = '0000_000000000_0005188', non_del_files = '1730', remaining_bytes = 738913280000 where label = 'VOV138'; 00:00:29 cdfenmvr192a.fnal.gov 006151 root I LTO4_92MV transfer complete volume=VOV138 location=5188 Thread tape_thread 00:00:29 last message repeated 1 times 00:00:29 stkensrv0n.fnal.gov 028762 enstore I VOLSRV Updating update volume set sum_wr_access = '1729', modification_time = '2009-11-04 00:00:29' where label = 'VOV138'; 00:00:29 cdfenmvr192a.fnal.gov 006151 root I LTO4_92MV Sending done to client: {'vc': {'comment': '', 'storage_group': 'ktev', 'blocksize': 131072, 'declared': 1254165875.0, 'library': 'CD-LTO4F1', 'si_time': [0.0, 0.0], 'user_inhibit': ['none', 'none'], 'system_inhibit': ['none', 'none'], 'external_label': 'VOV138', 'wrapper': 'cern', 'remaining_bytes': 738919424000L, 'sum_mounts': 1, 'sum_rd_access': 0, 'file_family_width': 1, 'last_access': 1257314423.0, 'media_type': 'LTO4', 'status': ('ok', None), 'eod_cookie': '0000_000000000_0005185', 'capacity_bytes': 858993459200L, 'non_del_files': 1729, 'sum_wr_err': 0, 'sum_wr_access': 1728, 'file_family': 'ktev_migration-MIGRATION', 'address': ('131.225.13.59', 7502), 'volume_family': 'ktev.ktev_migration-MIGRATION.cern', 'modification_time': 1257314423.0, 'write_protected': 'n', 'sum_rd_err': 0, 'first_access': 1257302588.0}, 'outfilepath': '/pnfs/fs/usr/Migration/ktev/migrated_from_dlt/KQ/KQRC/KQRC44/f6813', 'encp': {'agetime': 30, 'adminpri': 6, 'delpri': 1, 'basepri': 1000007, 'delayed_dismount': 2, 'curpri': 1000007}, 'file_size': 121592L, 'mover': {'max_buffer': 2621440000L, 'check_written_file': 600, 'max_failures': 5, 'library': 'CD-LTO4F1.library_manager', 'local_mover': 0, 'single_filemark': 1, 'max_consecutive_failures': 3, 'norestart': 'INQ', 'port': 7853, 'mc_device': '1,0,1,1', 'compression': 0, 'serial_num': '1310060800', 'update_interval_in_bound': 2, 'update_interval': 5, 'syslog_entry': 'sense|st[0-9]', 'status': ('ok', None), 'max_rate': 73400320.0, 'vendor_id': 'IBM', 'driver': 'FTTDriver', 'host': 'cdfenmvr192a', 'max_time_in_state': 1200, 'device': 'cdfenmvr192a:/dev/rmt/tps0d0n', 'name': 'LTO4_92.mover', 'send_stats': 1, 'product_id': 'ULTRIUM-TD4', 'statistics_path': '/tmp/enstore/enstore/LTO4_92.stat', 'mover_address': ('131.225.215.76', 7853), 'mount_delay': 15, 'check_first_written_file': 0, 'callback_addr': ('131.225.215.76', 60658), 'data_ip': 'cdfenmvr192a', 'logname': 'LTO4_92MV', 'hostip': '131.225.215.76', 'do_cleaning': 'No', 'media_changer': 'SL8500F1.media_changer'}, 'ignore_fair_share': 1, 'retry': None, 'lm': {'address': ('131.225.13.4', 8542)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fs/usr/ktev/migrated_from_dlt/KQ/KQRC/KQRC44/f6813', 'uid': 0, 'uname': 'root', 'mtime': 1257314410, 'rmajor': 0, 'machine': ('Linux', 'stkendm7a.fnal.gov', '2.6.9-89.0.9.ELsmp', '#1 SMP Mon Aug 24 13:47:43 CDT 2009', 'i686'), 'sanity_size': 65536, 'type': 'cern', 'pstat': (16877, 678574824L, 20L, 1, 0, 0, 512L, 1257314341, 1257314341, 1252942696), 'mode_octal': '0100016', 'mode': 33188, 'size_bytes': 121592L, 'gname': 'root', 'gid': 0, 'fullname': '/data/data2/Migration_Spool/VO6153:0000_000000000_0031889', 'inode': 675389792L, 'minor': 20}, 'version': 'v3_8 CVS $Revision: 1.963 $ encp.pyc', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/data/data2/Migration_Spool/VO6153:0000_000000000_0031889', 'status': ('ok', None), 'outfile': '/pnfs/fs/usr/Migration/ktev/migrated_from_dlt/KQ/KQRC/KQRC44/.(access)(00280000000000000841A160)', 'fc': {'uid': 0, 'external_label': 'VOV138', 'gid': 0, 'location_cookie': '0000_000000000_0005186', 'complete_crc': 2933529775L, 'bfid': 'CDMS125731442900000', 'sanity_cookie': (65536L, 1732236251L), 'size': 121592L}, 'infilepath': '/data/data2/Migration_Spool/VO6153:0000_000000000_0031889', 'at_the_top': 42, 'local_inode': 33308835L, 'resend': {'max_retry': 3, 'resubmits': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.13.19', 57444), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'encp_start_time': 1257314410.213377, 'seek_time': 0.026787996292114258, 'transfer_time': 5.070347785949707, 't0': 1257314410, 'mount_time': 0.018416166305541992, 'lm_dequeued': 1257314424.3975599, 'job_queued': 1257314410.5168271, 'drive_call_time': 0.011151790618896484, 'in_queue': 13.880478858947754, 'drive_transfer_time': 4.9850828647613525}, 'user_level': 'admin', 'unique_id': 'stkendm7a.fnal.gov-1257314410-18643-24633'} Thread tape_thread 00:00:29 cdfenmvr192a.fnal.gov 006151 root I LTO4_92MV Done is sent Thread tape_thread 00:00:29 cdfenmvr192a.fnal.gov 006151 root I LTO4_92MV maybe_clean Thread tape_thread 00:00:29 stkendm7a.fnal.gov 018643 root I ENCP Received final dialog for stkendm7a.fnal.gov-1257314410-18643-24633. elapsed=154676.000sec Thread WRITE 00:00:29 stkendm7a.fnal.gov 018643 root I ENCP leaving transfer_file(). Thread WRITE 00:00:29 cdfenmvr192a.fnal.gov 006151 root I LTO4_92MV returned from maybe_clean Thread tape_thread 00:00:29 stkensrv0n.fnal.gov 028796 enstore I FILSRV Updating update file set deleted = 'n', pnfs_path = '/pnfs/fs/usr/ktev/migrated_from_dlt/KQ/KQRC/KQRC44/f6813', drive = 'cdfenmvr192a:/dev/rmt/tps0d0n:1310060800', pnfs_id = '00280000000000000841A160' where bfid = 'CDMS125731442900000'; 00:00:29 stkenmvr195a.fnal.gov 021479 root I LTO4_95MV written bytes 2348017664/2348017664, blocks 17914 header 192 trailer 459 Thread tape_thread 00:00:29 stkensrv4n.fnal.gov 004149 enstore I SL8500G1MC dismount VOLUME 0,6,1,2 force => 0,Dismount: Forced dismount of VOT806 from 0, 6, 1, 2 00:00:29 stkendm7a.fnal.gov 018643 root I ENCP MSG_TYPE=ENCP_XFER write_to_hsm /data/data2/Migration_Spool/VO6153:0000_000000000_0031889 -> /pnfs/fs/usr/Migration/ktev/migrated_from_dlt/KQ/KQRC/KQRC44/f6813: 121592 bytes copied to VOV138 at 0.0228 MB/S (72 MB/S network) (0.0233 MB/S drive) (5.31 MB/S disk) mover=LTO4_92.mover drive_id=ULTRIUM-TD4 drive_sn=1310060800 drive_vendor=IBM elapsed=19.745 {'storage_group': 'ktev', 'encp_crc': 2933529775L, 'disk_rate': 5.3128914015832995, 'drive_rate': 0.02326123168386289, 'driver': 'FTTDriver', 'transfer_rate': 0.022844533519771521, 'mover_interface': 'cdfenmvr192a', 'overall_rate': 0.020979293662145584, 'media_changer': 'SL8500F1.media_changer', 'network_rate': 0.022844533519771521, 'encp_ip': '131.225.13.19', 'unique_id': 'stkendm7a.fnal.gov-1257314410-18643-24633'} Thread WRITE 00:00:29 stkendm7a.fnal.gov 018643 root I ENCP File /pnfs/fs/usr/Migration/ktev/migrated_from_dlt/KQ/KQRC/KQRC44/f6813 copied successfully. Thread WRITE 00:00:29 stkendm7a.fnal.gov 018643 root I ENCP Completed transferring 121592 bytes in 1 files in 19.7460889816 sec. Overall rate = 0.021 MB/sec. Transfer rate = 0.0228 MB/sec. Network rate = 72 MB/sec. Drive rate = 0.0233 MB/sec. Disk rate = 5.31 MB/sec. Exit status = 0. Thread WRITE 00:00:29 stkensrv4n.fnal.gov 003689 enstore I SL8500G1MC FINISHED dismount VOT806 0,6,1,2 returned ('ok', 0, 'dismount VOLUME 0,6,1,2 force => 0,Dismount: Forced dismount of VOT806 from 0, 6, 1, 2', '', '') 00:00:29 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:29 stkenmvr203a.fnal.gov 006116 root I LTO4_103MV dismounted VOT806 ULTRIUM-TD4 00:00:29 Thread media_thread 00:00:30 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM 00:00:30 stkensrv0n.fnal.gov 028796 enstore I FILSRV start modifying {'pnfsid': '00280000000000000841A160', 'pnfs_name0': '/pnfs/fs/usr/ktev/migrated_from_dlt/KQ/KQRC/KQRC44/f6813', 'deleted': 'no', 'external_label': 'VOV138', 'drive': 'cdfenmvr192a:/dev/rmt/tps0d0n:1310060800', 'update': '2009-11-04 00:00:29.68238', 'gid': 0, 'location_cookie': '0000_000000000_0005186', 'complete_crc': 2933529775L, 'uid': 0, 'bfid': 'CDMS125731442900000', 'sanity_cookie': (65536L, 1732236251L), 'size': 121592L} 00:00:30 stkensrv0n.fnal.gov 028796 enstore I FILSRV Updating update file set pnfs_id = '001F00000000000000F785F8' where bfid = 'CDMS125731442900000'; 00:00:30 stkensrv0n.fnal.gov 028796 enstore I FILSRV modified to {'pnfsid': '001F00000000000000F785F8', 'pnfs_name0': '/pnfs/fs/usr/ktev/migrated_from_dlt/KQ/KQRC/KQRC44/f6813', 'deleted': 'no', 'external_label': 'VOV138', 'drive': 'cdfenmvr192a:/dev/rmt/tps0d0n:1310060800', 'update': '2009-11-04 00:00:29.68238', 'gid': 0, 'location_cookie': '0000_000000000_0005186', 'complete_crc': 2933529775L, 'uid': 0, 'bfid': 'CDMS125731442900000', 'sanity_cookie': (65536L, 1732236251L), 'size': 121592L} 00:00:30 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:30 stkendm7a.fnal.gov 018643 root I ENCP Version: v3_8 CVS $Revision: 1.963 $ encp.pyc OS: Linux 2.6.9-89.0.9.ELsmp i686 Release: Scientific Linux Fermi LTS release 4.4 (Wilson) User: root(0) Group: root(0) Euser: root(0) Egroup: root(0) Library: CD-LTO4F1 Storage Group: ktev File Family: ktev_migration-MIGRATION FF Wrapper: cern FF Width: 1 Current working directory: stkendm7a.fnal.gov:/opt/enstore/src Command line: encp --delayed-dismount 2 --ignore-fair-share --threaded --priority 0 --storage-group ktev --file-family ktev_migration-MIGRATION --file-family-wrapper cern --library CD-LTO4F1 --override-path /pnfs/fs/usr/ktev/migrated_from_dlt/KQ/KQRC/KQRC37/f2692 --threaded /data/data2/Migration_Spool/VO6153:0000_000000000_0031898 /pnfs/fs/usr/Migration/ktev/migrated_from_dlt/KQ/KQRC/KQRC37/f2692 Thread WRITE 00:00:30 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM 00:00:30 stkendm7a.fnal.gov 018643 root I ENCP Creating zero length output files. Thread WRITE 00:00:30 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM write_to_hsm next work on vol=VOV138 mover=LTO4_92.mover requester:root 00:00:30 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM HAVE_BOUND:sending write_to_hsm /pnfs/fs/usr/ktev/migrated_from_dlt/KQ/KQRC/KQRC42/f1909 to mover LTO4_92.mover ('131.225.215.76', 7853) DEL_DISM 2 00:00:30 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM MSG_TYPE=ADD_TO_LMQ request added to work at movers queue for storage group : ktev 00:00:30 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM HAVE_BOUND:sending {'vc': {'storage_group': 'ktev', 'external_label': 'VOV138', 'library': 'CD-LTO4F1', 'file_family': 'ktev_migration-MIGRATION', 'wrapper': 'cern', 'address': ('131.225.13.59', 7502), 'volume_family': 'ktev.ktev_migration-MIGRATION.cern', 'file_family_width': 1}, 'outfilepath': '/pnfs/fs/usr/Migration/ktev/migrated_from_dlt/KQ/KQRC/KQRC42/f1909', 'encp': {'agetime': 30, 'adminpri': 6, 'delpri': 1, 'basepri': 1000007, 'delayed_dismount': 2, 'curpri': 1000007}, 'file_size': 6969932L, 'mover': 'LTO4_92.mover', 'ignore_fair_share': 1, 'retry': None, 'lm': {'address': ('131.225.13.4', 8542)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fs/usr/ktev/migrated_from_dlt/KQ/KQRC/KQRC42/f1909', 'uid': 0, 'uname': 'root', 'type': 'cern', 'gid': 0, 'mtime': 1257314417, 'rmajor': 0, 'machine': ('Linux', 'stkendm7a.fnal.gov', '2.6.9-89.0.9.ELsmp', '#1 SMP Mon Aug 24 13:47:43 CDT 2009', 'i686'), 'sanity_size': 65536, 'pstat': (16877, 678574128L, 20L, 1, 0, 0, 512L, 1257314299, 1257314299, 1252942651), 'mode_octal': '0100016', 'mode': 33188, 'gname': 'root', 'size_bytes': 6969932L, 'fullname': '/data/data2/Migration_Spool/VO6153:0000_000000000_0032558', 'inode': 675389816L, 'minor': 20}, 'version': 'v3_8 CVS $Revision: 1.963 $ encp.pyc', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/data/data2/Migration_Spool/VO6153:0000_000000000_0032558', 'status': ('ok', None), 'outfile': '/pnfs/fs/usr/Migration/ktev/migrated_from_dlt/KQ/KQRC/KQRC42/.(access)(00280000000000000841A178)', 'fc': {'size': 6969932L, 'external_label': 'VOV138', 'pnfsid': '00280000000000000841A178', 'address': ('131.225.13.59', 7501)}, 'infilepath': '/data/data2/Migration_Spool/VO6153:0000_000000000_0032558', 'at_the_top': 39, 'local_inode': 4495421L, 'resend': {'max_retry': 3, 'resubmits': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.13.19', 57448), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'lm_dequeued': 1257314430.407619, 'encp_start_time': 1257314417.5619681, 'job_queued': 1257314417.7896371, 't0': 1257314417, 'in_queue': 12.617725849151611}, 'user_level': 'admin', 'unique_id': 'stkendm7a.fnal.gov-1257314417-18643-24634'} to mover 00:00:30 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM 00:00:30 cdfenmvr192a.fnal.gov 006151 root I LTO4_92MV WRITE_TO_HSM Thread MainThread 00:00:30 cdfenmvr192a.fnal.gov 006151 root I LTO4_92MV SENDING {'vc': {'storage_group': 'ktev', 'external_label': 'VOV138', 'library': 'CD-LTO4F1', 'file_family': 'ktev_migration-MIGRATION', 'wrapper': 'cern', 'address': ('131.225.13.59', 7502), 'volume_family': 'ktev.ktev_migration-MIGRATION.cern', 'file_family_width': 1}, 'outfilepath': '/pnfs/fs/usr/Migration/ktev/migrated_from_dlt/KQ/KQRC/KQRC42/f1909', 'encp': {'agetime': 30, 'adminpri': 6, 'delpri': 1, 'basepri': 1000007, 'delayed_dismount': 2, 'curpri': 1000007}, 'file_size': 6969932L, 'mover': {'max_buffer': 2621440000L, 'check_written_file': 600, 'max_failures': 5, 'library': 'CD-LTO4F1.library_manager', 'local_mover': 0, 'single_filemark': 1, 'max_consecutive_failures': 3, 'norestart': 'INQ', 'port': 7853, 'mc_device': '1,0,1,1', 'compression': 0, 'serial_num': '1310060800', 'update_interval_in_bound': 2, 'update_interval': 5, 'syslog_entry': 'sense|st[0-9]', 'status': ('ok', None), 'max_rate': 73400320.0, 'vendor_id': 'IBM', 'driver': 'FTTDriver', 'host': 'cdfenmvr192a', 'max_time_in_state': 1200, 'device': 'cdfenmvr192a:/dev/rmt/tps0d0n', 'name': 'LTO4_92.mover', 'send_stats': 1, 'product_id': 'ULTRIUM-TD4', 'statistics_path': '/tmp/enstore/enstore/LTO4_92.stat', 'mover_address': ('131.225.215.76', 7853), 'mount_delay': 15, 'check_first_written_file': 0, 'callback_addr': ('131.225.215.76', 60660), 'data_ip': 'cdfenmvr192a', 'logname': 'LTO4_92MV', 'hostip': '131.225.215.76', 'do_cleaning': 'No', 'media_changer': 'SL8500F1.media_changer'}, 'ignore_fair_share': 1, 'retry': None, 'lm': {'address': ('131.225.13.4', 8542)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fs/usr/ktev/migrated_from_dlt/KQ/KQRC/KQRC42/f1909', 'uid': 0, 'uname': 'root', 'mtime': 1257314417, 'rmajor': 0, 'machine': ('Linux', 'stkendm7a.fnal.gov', '2.6.9-89.0.9.ELsmp', '#1 SMP Mon Aug 24 13:47:43 CDT 2009', 'i686'), 'sanity_size': 65536, 'type': 'cern', 'pstat': (16877, 678574128L, 20L, 1, 0, 0, 512L, 1257314299, 1257314299, 1252942651), 'mode_octal': '0100016', 'mode': 33188, 'size_bytes': 6969932L, 'gname': 'root', 'gid': 0, 'fullname': '/data/data2/Migration_Spool/VO6153:0000_000000000_0032558', 'inode': 675389816L, 'minor': 20}, 'version': 'v3_8 CVS $Revision: 1.963 $ encp.pyc', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/data/data2/Migration_Spool/VO6153:0000_000000000_0032558', 'status': ('ok', None), 'outfile': '/pnfs/fs/usr/Migration/ktev/migrated_from_dlt/KQ/KQRC/KQRC42/.(access)(00280000000000000841A178)', 'fc': {'size': 6969932L, 'external_label': 'VOV138', 'pnfsid': '00280000000000000841A178', 'address': ('131.225.13.59', 7501)}, 'infilepath': '/data/data2/Migration_Spool/VO6153:0000_000000000_0032558', 'at_the_top': 39, 'local_inode': 4495421L, 'resend': {'max_retry': 3, 'resubmits': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.13.19', 57448), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'lm_dequeued': 1257314430.407619, 'encp_start_time': 1257314417.5619681, 'job_queued': 1257314417.7896371, 't0': 1257314417, 'in_queue': 12.617725849151611}, 'user_level': 'admin', 'unique_id': 'stkendm7a.fnal.gov-1257314417-18643-24634'} Thread client_connect_thread 00:00:30 stkendm7a.fnal.gov 018643 root I ENCP Listening for control socket at: ('131.225.13.19', 57448) Thread WRITE_1 00:00:30 stkendm7a.fnal.gov 018643 root I ENCP Waiting for mover to connect control socket at address: ('131.225.13.19', 57448) elapsed=154676.870sec Thread WRITE_1 00:00:30 stkendm7a.fnal.gov 018643 root I ENCP Control socket ('131.225.13.19', 57448) is connected to ('131.225.215.76', 60661) for stkendm7a.fnal.gov-1257314417-18643-24634. elapsed=154676.870sec Thread WRITE_1 00:00:30 stkendm7a.fnal.gov 018643 root I ENCP Received callback ticket from mover LTO4_92.mover for transfer stkendm7a.fnal.gov-1257314417-18643-24634. elapsed=154676.880sec Thread WRITE_1 00:00:30 stkendm7a.fnal.gov 018643 root I ENCP Attempting to connect data socket to mover at ('131.225.215.76', 60660). Thread WRITE_1 00:00:30 stkendm7a.fnal.gov 018643 root I ENCP Connecting data socket to mover (('131.225.215.76', 60660)) with interface 131.225.13.19. elapsed=154676.880sec Thread WRITE_1 00:00:30 stkendm7a.fnal.gov 018643 root I ENCP Data socket ('131.225.13.19', 57454) is connected to ('131.225.215.76', 60660). elapsed=154676.880sec Thread WRITE_1 00:00:30 stkendm7a.fnal.gov 018643 root I ENCP Opening local file. Thread WRITE_1 00:00:30 stkendm7a.fnal.gov 018643 root I ENCP pf_status = ('ok', None) skip_layer_cleanup = False Thread WRITE_1 00:00:30 stkendm7a.fnal.gov 018643 root I ENCP Sending /pnfs/fs/usr/Migration/ktev/migrated_from_dlt/KQ/KQRC/KQRC37/f2692 write request to LM: unique_id: stkendm7a.fnal.gov-1257314430-18643-24636 inputfile: /data/data2/Migration_Spool/VO6153:0000_000000000_0031898 outputfile: /pnfs/fs/usr/Migration/ktev/migrated_from_dlt/KQ/KQRC/KQRC37/.(access)(00280000000000000841A1C0) Thread WRITE 00:00:30 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM MSG_TYPE=ADD_TO_LMQ request added to pending queue for storage group : ktev 00:00:30 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM write Q'd /data/data2/Migration_Spool/VO6153:0000_000000000_0031898 (stkendm7a.fnal.gov-1257314430-18643-24636) -> /pnfs/fs/usr/ktev/migrated_from_dlt/KQ/KQRC/KQRC37/f2692 : library=CD-LTO4F1 family=ktev_migration-MIGRATION requester:root volume_family:ktev.ktev_migration-MIGRATION.cern 00:00:30 stkenmvr197a.fnal.gov 025501 root I LTO4_97MV filemarks written. Tape VON178 absolute location in blocks 4680304 Thread tape_thread 00:00:30 stkenmvr197a.fnal.gov 025501 root I LTO4_97MV write_tape timing:pull 0.136166810989 write 5.03421139717 crc_check 7.41385197639 freespace 0.153527975082 block_write 12.7377581596 idle 0.248347043991 Thread tape_thread 00:00:30 stkenmvr197a.fnal.gov 025501 root I LTO4_97MV drive stats after write. Tape VON178 position 790 block 0 block_size 0 bloc_loc 4680304 tot_blocks 762939000 BOT 0 read_err 0 write_err 0 bytes 1594721280 block_write_tot 12.7377581596 tape_rate 125196385.425 Thread tape_thread 00:00:30 stkenmvr197a.fnal.gov 025501 root I LTO4_97MV remaining bytes info in DB 200359936000 estimated 198765214720 reported from drive 198881280000 position 791 Thread tape_thread 00:00:30 stkenmvr197a.fnal.gov 025501 root I LTO4_97MV new bitfile request {'uid': 0, 'external_label': 'VON178', 'gid': 0, 'location_cookie': '0000_000000000_0000790', 'complete_crc': 3280895055L, 'sanity_cookie': (65536L, 624490572L), 'size': 1594720974L} Thread tape_thread 00:00:30 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:30 cdfenmvr192a.fnal.gov 006151 root I LTO4_92MV tape position: target 5188 current 5188 0 Thread seek_thread 00:00:30 cdfenmvr192a.fnal.gov 006151 root I LTO4_92MV write_tape starting, bytes_to_write=6970368 Thread tape_thread 00:00:30 cdfenmvr192a.fnal.gov 006151 root I LTO4_92MV Write starting. Tape VOV138 absolute location in blocks 255470 Thread tape_thread 00:00:30 stkenmvr197a.fnal.gov 025501 root I LTO4_97MV set remaining: VON178 198881280000 0000_000000000_0000791 Thread tape_thread 00:00:30 last message repeated 1 times 00:00:30 stkensrv0n.fnal.gov 028762 enstore I VOLSRV Updating update volume set last_access = '2009-11-04 00:00:30', eod_cookie = '0000_000000000_0000791', non_del_files = '811', remaining_bytes = 198881280000 where label = 'VON178'; 00:00:30 stkendm7a.fnal.gov 018643 root I ENCP File /data/data2/Migration_Spool/VO6153:0000_000000000_0032558 transfered for stkendm7a.fnal.gov-1257314417-18643-24634 in 0.0916419029236 seconds. Thread WRITE_1 00:00:30 stkenmvr197a.fnal.gov 025501 root I LTO4_97MV transfer complete volume=VON178 location=791 Thread tape_thread 00:00:30 last message repeated 1 times 00:00:30 stkensrv0n.fnal.gov 028762 enstore I VOLSRV Updating update volume set sum_wr_access = '791', modification_time = '2009-11-04 00:00:30' where label = 'VON178'; 00:00:30 stkenmvr197a.fnal.gov 025501 root I LTO4_97MV Sending done to client: {'vc': {'comment': ' ', 'storage_group': 'minos', 'blocksize': 131072, 'declared': 1225477350.0, 'library': 'CD-LTO4G1', 'si_time': [1242756109.0, 0.0], 'user_inhibit': ['none', 'none'], 'system_inhibit': ['none', 'none'], 'external_label': 'VON178', 'wrapper': 'cpio_odc', 'remaining_bytes': 200359936000L, 'sum_mounts': 495, 'sum_rd_access': 1524, 'file_family_width': 1, 'last_access': 1257071999.0, 'media_type': 'LTO4', 'status': ('ok', None), 'eod_cookie': '0000_000000000_0000790', 'capacity_bytes': 858993459200L, 'non_del_files': 810, 'sum_wr_err': 1, 'sum_wr_access': 790, 'file_family': 'reco_near_dogwood0_sntp', 'address': ('131.225.13.59', 7502), 'volume_family': 'minos.reco_near_dogwood0_sntp.cpio_odc', 'modification_time': 1257071999.0, 'write_protected': 'n', 'sum_rd_err': 1, 'first_access': 1240646517.0}, 'outfilepath': '/pnfs/fnal.gov/usr/minos/reco_near/dogwood0/sntp_data/2009-10/N00017122_0000.spill.sntp.dogwood0.0.root', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 1, 'delayed_dismount': None, 'curpri': 1}, 'file_size': 1594720974L, 'mover': {'max_buffer': 2621440000L, 'check_written_file': 600, 'max_failures': 5, 'library': 'CD-LTO4G1.library_manager', 'local_mover': 0, 'single_filemark': 1, 'max_consecutive_failures': 3, 'norestart': 'INQ', 'port': 7858, 'mc_device': '0,7,1,13', 'compression': 0, 'serial_num': '9310062655', 'update_interval_in_bound': 2, 'update_interval': 5, 'syslog_entry': 'sense|st[0-9]', 'status': ('ok', None), 'max_rate': 73400320.0, 'vendor_id': 'IBM', 'driver': 'FTTDriver', 'host': 'stkenmvr197a', 'max_time_in_state': 1200, 'device': 'stkenmvr197a:/dev/rmt/tps0d0n', 'name': 'LTO4_97.mover', 'send_stats': 1, 'product_id': 'ULTRIUM-TD4', 'statistics_path': '/tmp/enstore/enstore/LTO4_97.stat', 'mover_address': ('131.225.13.192', 7858), 'mount_delay': 15, 'check_first_written_file': 0, 'callback_addr': ('131.225.13.192', 37244), 'data_ip': 'stkenmvr197a', 'logname': 'LTO4_97MV', 'hostip': '131.225.13.192', 'do_cleaning': 'No', 'media_changer': 'SL8500G1.media_changer'}, 'ignore_fair_share': None, 'retry': None, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fnal.gov/usr/minos/reco_near/dogwood0/sntp_data/2009-10/N00017122_0000.spill.sntp.dogwood0.0.root', 'uid': 0, 'uname': 'root', 'mtime': 1257314235, 'rmajor': 0, 'machine': ('Linux', 'stkendca22a.fnal.gov', '2.6.9-89.0.9.ELsmp', '#1 SMP Mon Aug 24 13:47:43 CDT 2009', 'i686'), 'sanity_size': 65536, 'type': 'cpio_odc', 'pstat': (16893, 252364496L, 20L, 1, 42411, 5111, 512L, 1257299891, 1257299891, 1254755314), 'mode_octal': '0100036', 'mode': 33188, 'size_bytes': 1594720974L, 'gname': 'root', 'gid': 0, 'fullname': '/diskb/write-pool-2/data/000F0000000000000B20C180', 'inode': 253804928L, 'minor': 20}, 'version': 'v3_8b CVS $Revision: 1.922 $ ', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/diskb/write-pool-2/data/000F0000000000000B20C180', 'status': ('ok', None), 'outfile': '/pnfs/fnal.gov/usr/minos/reco_near/dogwood0/sntp_data/2009-10/.(access)(000F0000000000000B20C180)', 'fc': {'uid': 0, 'external_label': 'VON178', 'gid': 0, 'location_cookie': '0000_000000000_0000790', 'complete_crc': 3280895055L, 'bfid': 'CDMS125731443000000', 'sanity_cookie': (65536L, 624490572L), 'size': 1594720974L}, 'infilepath': '/diskb/write-pool-2/data/000F0000000000000B20C180', 'at_the_top': 1, 'local_inode': 1073746739L, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.13.100', 51702), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'encp_start_time': 1257314234.5652289, 'seek_time': 172.42301201820374, 'transfer_time': 21.514127969741821, 't0': 1257314234, 'lm_dequeued': 1257314236.684629, 'job_queued': 1257314235.4107909, 'drive_call_time': 4.870556116104126, 'in_queue': 1.2735400199890137, 'drive_transfer_time': 14.827658891677856}, 'user_level': 'user2', 'unique_id': 'stkendca22a.fnal.gov-1257314235-10203-0'} Thread tape_thread 00:00:30 stkenmvr197a.fnal.gov 025501 root I LTO4_97MV Done is sent Thread tape_thread 00:00:30 stkenmvr197a.fnal.gov 025501 root I LTO4_97MV maybe_clean Thread tape_thread 00:00:30 stkendca22a.fnal.gov 010203 root I ENCP Received final dialog for stkendca22a.fnal.gov-1257314235-10203-0. elapsed=196.130sec 00:00:30 stkendca22a.fnal.gov 010203 root I ENCP leaving transfer_file(). 00:00:30 stkenmvr197a.fnal.gov 025501 root I LTO4_97MV returned from maybe_clean Thread tape_thread 00:00:30 stkendca22a.fnal.gov 010203 root I ENCP pf_status = ('ok', None) skip_layer_cleanup = False 00:00:30 stkensrv0n.fnal.gov 028796 enstore I FILSRV Updating update file set deleted = 'n', pnfs_path = '/pnfs/fnal.gov/usr/minos/reco_near/dogwood0/sntp_data/2009-10/N00017122_0000.spill.sntp.dogwood0.0.root', drive = 'stkenmvr197a:/dev/rmt/tps0d0n:9310062655', pnfs_id = '000F0000000000000B20C180' where bfid = 'CDMS125731443000000'; 00:00:31 stkendca22a.fnal.gov 010203 root I ENCP MSG_TYPE=ENCP_XFER write_to_hsm /diskb/write-pool-2/data/000F0000000000000B20C180 -> /pnfs/fnal.gov/usr/minos/reco_near/dogwood0/sntp_data/2009-10/N00017122_0000.spill.sntp.dogwood0.0.root: 1594720974 bytes copied to VON178 at 70.7 MB/S (78.9 MB/S network) (103 MB/S drive) (77.6 MB/S disk) mover=LTO4_97.mover drive_id=ULTRIUM-TD4 drive_sn=9310062655 drive_vendor=IBM elapsed=196.48 {'storage_group': 'minos', 'encp_crc': 3280895055L, 'disk_rate': 77.599199229692232, 'drive_rate': 102.56807536837427, 'driver': 'FTTDriver', 'transfer_rate': 70.669055994120725, 'mover_interface': 'stkenmvr197a', 'overall_rate': 7.8262437439213306, 'media_changer': 'SL8500G1.media_changer', 'network_rate': 70.669055994120725, 'encp_ip': '131.225.13.100', 'unique_id': 'stkendca22a.fnal.gov-1257314235-10203-0'} 00:00:31 stkendca22a.fnal.gov 010203 root I ENCP File /pnfs/fnal.gov/usr/minos/reco_near/dogwood0/sntp_data/2009-10/N00017122_0000.spill.sntp.dogwood0.0.root copied successfully. 00:00:31 stkendca22a.fnal.gov 010203 root I ENCP Completed transferring 1594720974 bytes in 1 files in 196.489500046 sec. Overall rate = 7.83 MB/sec. Transfer rate = 70.7 MB/sec. Network rate = 78.9 MB/sec. Drive rate = 103 MB/sec. Disk rate = 77.6 MB/sec. Exit status = 0. 00:00:31 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:31 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM write_to_hsm next work on vol=VON178 mover=LTO4_97.mover requester:root 00:00:31 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM HAVE_BOUND:sending write_to_hsm /pnfs/fnal.gov/usr/minos/reco_near/dogwood0/sntp_data/2009-10/N00017122_0000.cosmic.sntp.dogwood0.0.root to mover LTO4_97.mover ('131.225.13.192', 7858) DEL_DISM None 00:00:31 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM MSG_TYPE=ADD_TO_LMQ request added to work at movers queue for storage group : minos 00:00:31 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM HAVE_BOUND:sending {'vc': {'storage_group': 'minos', 'external_label': 'VON178', 'library': 'CD-LTO4G1', 'file_family': 'reco_near_dogwood0_sntp', 'wrapper': 'cpio_odc', 'address': ('131.225.13.59', 7502), 'volume_family': 'minos.reco_near_dogwood0_sntp.cpio_odc', 'file_family_width': 1}, 'outfilepath': '/pnfs/fnal.gov/usr/minos/reco_near/dogwood0/sntp_data/2009-10/N00017122_0000.cosmic.sntp.dogwood0.0.root', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 1, 'delayed_dismount': None, 'curpri': 1}, 'file_size': 289487922L, 'mover': 'LTO4_97.mover', 'ignore_fair_share': None, 'retry': None, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fnal.gov/usr/minos/reco_near/dogwood0/sntp_data/2009-10/N00017122_0000.cosmic.sntp.dogwood0.0.root', 'uid': 0, 'uname': 'root', 'type': 'cpio_odc', 'gid': 0, 'mtime': 1257314235, 'rmajor': 0, 'machine': ('Linux', 'stkendca22a.fnal.gov', '2.6.9-89.0.9.ELsmp', '#1 SMP Mon Aug 24 13:47:43 CDT 2009', 'i686'), 'sanity_size': 65536, 'pstat': (16893, 252364496L, 20L, 1, 42411, 5111, 512L, 1257299891, 1257299891, 1254755314), 'mode_octal': '0100036', 'mode': 33188, 'gname': 'root', 'size_bytes': 289487922L, 'fullname': '/diskb/write-pool-2/data/000F0000000000000B20C088', 'inode': 253804680L, 'minor': 20}, 'version': 'v3_8b CVS $Revision: 1.922 $ ', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/diskb/write-pool-2/data/000F0000000000000B20C088', 'status': ('ok', None), 'outfile': '/pnfs/fnal.gov/usr/minos/reco_near/dogwood0/sntp_data/2009-10/.(access)(000F0000000000000B20C088)', 'fc': {'size': 289487922L, 'external_label': 'VON178', 'address': ('131.225.13.59', 7501)}, 'infilepath': '/diskb/write-pool-2/data/000F0000000000000B20C088', 'at_the_top': 292, 'local_inode': 1073746733L, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.13.100', 51703), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'lm_dequeued': 1257314431.1667449, 'encp_start_time': 1257314234.6061749, 'job_queued': 1257314235.4269929, 't0': 1257314234, 'in_queue': 195.73944115638733}, 'user_level': 'user2', 'unique_id': 'stkendca22a.fnal.gov-1257314235-10240-0'} to mover 00:00:31 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:31 stkenmvr197a.fnal.gov 025501 root I LTO4_97MV WRITE_TO_HSM Thread MainThread 00:00:31 stkenmvr197a.fnal.gov 025501 root I LTO4_97MV SENDING {'vc': {'storage_group': 'minos', 'external_label': 'VON178', 'library': 'CD-LTO4G1', 'file_family': 'reco_near_dogwood0_sntp', 'wrapper': 'cpio_odc', 'address': ('131.225.13.59', 7502), 'volume_family': 'minos.reco_near_dogwood0_sntp.cpio_odc', 'file_family_width': 1}, 'outfilepath': '/pnfs/fnal.gov/usr/minos/reco_near/dogwood0/sntp_data/2009-10/N00017122_0000.cosmic.sntp.dogwood0.0.root', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 1, 'delayed_dismount': None, 'curpri': 1}, 'file_size': 289487922L, 'mover': {'max_buffer': 2621440000L, 'check_written_file': 600, 'max_failures': 5, 'library': 'CD-LTO4G1.library_manager', 'local_mover': 0, 'single_filemark': 1, 'max_consecutive_failures': 3, 'norestart': 'INQ', 'port': 7858, 'mc_device': '0,7,1,13', 'compression': 0, 'serial_num': '9310062655', 'update_interval_in_bound': 2, 'update_interval': 5, 'syslog_entry': 'sense|st[0-9]', 'status': ('ok', None), 'max_rate': 73400320.0, 'vendor_id': 'IBM', 'driver': 'FTTDriver', 'host': 'stkenmvr197a', 'max_time_in_state': 1200, 'device': 'stkenmvr197a:/dev/rmt/tps0d0n', 'name': 'LTO4_97.mover', 'send_stats': 1, 'product_id': 'ULTRIUM-TD4', 'statistics_path': '/tmp/enstore/enstore/LTO4_97.stat', 'mover_address': ('131.225.13.192', 7858), 'mount_delay': 15, 'check_first_written_file': 0, 'callback_addr': ('131.225.13.192', 37253), 'data_ip': 'stkenmvr197a', 'logname': 'LTO4_97MV', 'hostip': '131.225.13.192', 'do_cleaning': 'No', 'media_changer': 'SL8500G1.media_changer'}, 'ignore_fair_share': None, 'retry': None, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fnal.gov/usr/minos/reco_near/dogwood0/sntp_data/2009-10/N00017122_0000.cosmic.sntp.dogwood0.0.root', 'uid': 0, 'uname': 'root', 'mtime': 1257314235, 'rmajor': 0, 'machine': ('Linux', 'stkendca22a.fnal.gov', '2.6.9-89.0.9.ELsmp', '#1 SMP Mon Aug 24 13:47:43 CDT 2009', 'i686'), 'sanity_size': 65536, 'type': 'cpio_odc', 'pstat': (16893, 252364496L, 20L, 1, 42411, 5111, 512L, 1257299891, 1257299891, 1254755314), 'mode_octal': '0100036', 'mode': 33188, 'size_bytes': 289487922L, 'gname': 'root', 'gid': 0, 'fullname': '/diskb/write-pool-2/data/000F0000000000000B20C088', 'inode': 253804680L, 'minor': 20}, 'version': 'v3_8b CVS $Revision: 1.922 $ ', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/diskb/write-pool-2/data/000F0000000000000B20C088', 'status': ('ok', None), 'outfile': '/pnfs/fnal.gov/usr/minos/reco_near/dogwood0/sntp_data/2009-10/.(access)(000F0000000000000B20C088)', 'fc': {'size': 289487922L, 'external_label': 'VON178', 'address': ('131.225.13.59', 7501)}, 'infilepath': '/diskb/write-pool-2/data/000F0000000000000B20C088', 'at_the_top': 292, 'local_inode': 1073746733L, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.13.100', 51703), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'lm_dequeued': 1257314431.1667449, 'encp_start_time': 1257314234.6061749, 'job_queued': 1257314235.4269929, 't0': 1257314234, 'in_queue': 195.73944115638733}, 'user_level': 'user2', 'unique_id': 'stkendca22a.fnal.gov-1257314235-10240-0'} Thread client_connect_thread 00:00:31 stkendca22a.fnal.gov 010240 root I ENCP Control socket ('131.225.13.100', 51703) is connected to ('131.225.13.192', 37254) for stkendca22a.fnal.gov-1257314235-10240-0. elapsed=196.600sec 00:00:31 stkendca22a.fnal.gov 010240 root I ENCP Received callback ticket from mover LTO4_97.mover for transfer stkendca22a.fnal.gov-1257314235-10240-0. elapsed=196.600sec 00:00:31 stkendca22a.fnal.gov 010240 root I ENCP Atempting to connect data socket to mover at ('131.225.13.192', 37253). 00:00:31 stkendca22a.fnal.gov 010240 root I ENCP Connecting data socket to mover (('131.225.13.192', 37253)) with interface 131.225.13.100. elapsed=196.600sec 00:00:31 stkendca22a.fnal.gov 010240 root I ENCP Data socket ('131.225.13.100', 51748) is connected to ('131.225.13.192', 37253). elapsed=196.600sec 00:00:31 stkendca22a.fnal.gov 010240 root I ENCP Opening local file. 00:00:31 stkenmvr197a.fnal.gov 025501 root I LTO4_97MV tape position: target 791 current 791 0 Thread seek_thread 00:00:31 stkenmvr197a.fnal.gov 025501 root I LTO4_97MV write_tape starting, bytes_to_write=289488384 Thread tape_thread 00:00:31 stkendca22a.fnal.gov 010240 root I ENCP pf_status = ('ok', None) skip_layer_cleanup = False 00:00:31 stkenmvr197a.fnal.gov 025501 root I LTO4_97MV Write starting. Tape VON178 absolute location in blocks 4680304 Thread tape_thread 00:00:31 stkenmvr195a.fnal.gov 021479 root I LTO4_95MV filemarks written. Tape VOR957 absolute location in blocks 3953262 Thread tape_thread 00:00:31 stkenmvr195a.fnal.gov 021479 root I LTO4_95MV write_tape timing:pull 0.21525478363 write 6.45865678787 crc_check 11.0049307346 freespace 0.194721460342 block_write 17.8735637665 idle 0.346119880676 Thread tape_thread 00:00:31 stkenmvr195a.fnal.gov 021479 root I LTO4_95MV drive stats after write. Tape VOR957 position 171 block 0 block_size 0 bloc_loc 3953262 tot_blocks 762939000 BOT 0 read_err 0 write_err 0 bytes 2348017664 block_write_tot 17.8735637665 tape_rate 131368186.819 Thread tape_thread 00:00:31 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM 00:00:31 stkenmvr195a.fnal.gov 021479 root I LTO4_95MV remaining bytes info in DB 299365376000 estimated 297017358336 reported from drive 297185280000 position 172 Thread tape_thread 00:00:31 stkenmvr195a.fnal.gov 021479 root I LTO4_95MV new bitfile request {'uid': 0, 'external_label': 'VOR957', 'gid': 0, 'location_cookie': '0000_000000000_0000171', 'complete_crc': 2230142374L, 'sanity_cookie': (65536L, 1781711703L), 'size': 2348017013L} Thread tape_thread 00:00:31 stkenmvr207a.fnal.gov 006260 root I LTO4_107MV written bytes 3957460480/3957460480, blocks 30194 header 192 trailer 412 Thread tape_thread 00:00:31 stkenmvr195a.fnal.gov 021479 root I LTO4_95MV set remaining: VOR957 297185280000 0000_000000000_0000172 Thread tape_thread 00:00:31 last message repeated 1 times 00:00:31 stkensrv0n.fnal.gov 028762 enstore I VOLSRV Updating update volume set last_access = '2009-11-04 00:00:31', eod_cookie = '0000_000000000_0000172', non_del_files = '171', remaining_bytes = 297185280000 where label = 'VOR957'; 00:00:31 stkenmvr195a.fnal.gov 021479 root I LTO4_95MV transfer complete volume=VOR957 location=172 Thread tape_thread 00:00:31 last message repeated 1 times 00:00:31 stkensrv0n.fnal.gov 028762 enstore I VOLSRV Updating update volume set sum_wr_access = '171', modification_time = '2009-11-04 00:00:31' where label = 'VOR957'; 00:00:31 stkenmvr195a.fnal.gov 021479 root I LTO4_95MV Sending done to client: {'vc': {'comment': '', 'storage_group': 'cms', 'blocksize': 131072, 'declared': 1255359326.0, 'library': 'CD-LTO4G1', 'si_time': [0.0, 1250983192.0], 'user_inhibit': ['none', 'none'], 'system_inhibit': ['none', 'none'], 'external_label': 'VOR957', 'wrapper': 'cpio_odc', 'remaining_bytes': 299365376000L, 'sum_mounts': 74, 'sum_rd_access': 0, 'file_family_width': 10, 'last_access': 1257314404.0, 'media_type': 'LTO4', 'status': ('ok', None), 'eod_cookie': '0000_000000000_0000171', 'capacity_bytes': 858993459200L, 'non_del_files': 170, 'sum_wr_err': 0, 'sum_wr_access': 170, 'file_family': 'cms11', 'address': ('131.225.13.59', 7502), 'volume_family': 'cms.cms11.cpio_odc', 'modification_time': 1257314404.0, 'write_protected': 'n', 'sum_rd_err': 0, 'first_access': 1257307290.0}, 'outfilepath': '/pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0080/58AAEAC1-BBC8-DE11-93CC-001731A28461.root', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 25, 'delayed_dismount': None, 'curpri': 25}, 'file_size': 2348017013L, 'mover': {'max_buffer': 2621440000L, 'check_written_file': 600, 'max_failures': 5, 'library': 'CD-LTO4G1.library_manager', 'local_mover': 0, 'single_filemark': 1, 'max_consecutive_failures': 3, 'norestart': 'INQ', 'port': 7856, 'mc_device': '0,7,1,14', 'compression': 0, 'serial_num': '1310155266', 'update_interval_in_bound': 2, 'update_interval': 5, 'syslog_entry': 'sense|st[0-9]', 'status': ('ok', None), 'max_rate': 73400320.0, 'vendor_id': 'IBM', 'driver': 'FTTDriver', 'host': 'stkenmvr195a', 'max_time_in_state': 1200, 'device': 'stkenmvr195a:/dev/rmt/tps0d0n', 'name': 'LTO4_95.mover', 'send_stats': 1, 'product_id': 'ULTRIUM-TD4', 'statistics_path': '/tmp/enstore/enstore/LTO4_95.stat', 'mover_address': ('131.225.13.190', 7856), 'mount_delay': 15, 'check_first_written_file': 0, 'callback_addr': ('131.225.13.190', 41180), 'data_ip': 'stkenmvr195a', 'logname': 'LTO4_95MV', 'hostip': '131.225.13.190', 'do_cleaning': 'No', 'media_changer': 'SL8500G1.media_changer'}, 'ignore_fair_share': None, 'retry': None, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0080/58AAEAC1-BBC8-DE11-93CC-001731A28461.root', 'uid': 0, 'uname': 'root', 'mtime': 1257312799, 'rmajor': 0, 'machine': ('Linux', 'cmsstor206.fnal.gov', '2.6.31', '#1 SMP Mon Sep 14 09:43:20 CDT 2009', 'x86_64'), 'sanity_size': 65536, 'type': 'cpio_odc', 'pstat': (16893, 120129424, 21L, 1, 9811, 5063, 512, 1257290009, 1257290009, 1257279324), 'mode_octal': '0100036', 'mode': 33188, 'size_bytes': 2348017013L, 'gname': 'root', 'gid': 0, 'fullname': '/storage/data2/write-pool-2/data/0007000000000000622F1F78', 'inode': 120528760L, 'minor': 21}, 'version': 'v3_7d CVS $Revision: 1.922 $ encp', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/storage/data2/write-pool-2/data/0007000000000000622F1F78', 'status': ('ok', None), 'outfile': '/pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0080/.(access)(0007000000000000622F1F78)', 'fc': {'uid': 0, 'external_label': 'VOR957', 'gid': 0, 'location_cookie': '0000_000000000_0000171', 'complete_crc': 2230142374L, 'bfid': 'CDMS125731443100000', 'sanity_cookie': (65536L, 1781711703L), 'size': 2348017013L}, 'infilepath': '/storage/data2/write-pool-2/data/0007000000000000622F1F78', 'at_the_top': 16, 'local_inode': 6839, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.191.77', 49362), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'encp_start_time': 1257312798.982126, 'seek_time': 0.029460906982421875, 'transfer_time': 26.172715902328491, 't0': 1257312798, 'mount_time': 0.016766071319580078, 'lm_dequeued': 1257314405.2625401, 'job_queued': 1257312799.8738201, 'drive_call_time': 6.2006313800811768, 'in_queue': 1605.3883938789368, 'drive_transfer_time': 20.001834869384766}, 'user_level': 'admin', 'unique_id': 'cmsstor206.fnal.gov-1257312799-1751-0'} Thread tape_thread 00:00:31 stkenmvr195a.fnal.gov 021479 root I LTO4_95MV Done is sent Thread tape_thread 00:00:31 stkenmvr195a.fnal.gov 021479 root I LTO4_95MV maybe_clean Thread tape_thread 00:00:31 cmsstor206.fnal.gov 001751 root I ENCP Received final dialog for cmsstor206.fnal.gov-1257312799-1751-0. elapsed=1632.600sec 00:00:31 cmsstor206.fnal.gov 001751 root I ENCP leaving transfer_file(). 00:00:31 stkenmvr195a.fnal.gov 021479 root I LTO4_95MV returned from maybe_clean Thread tape_thread 00:00:31 cmsstor206.fnal.gov 001751 root I ENCP pf_status = ('ok', None) skip_layer_cleanup = False 00:00:31 cdfenmvr192a.fnal.gov 006151 root I LTO4_92MV written bytes 6970368/6970368, blocks 54 header 0 trailer 436 Thread tape_thread 00:00:31 stkensrv0n.fnal.gov 028796 enstore I FILSRV Updating update file set deleted = 'n', pnfs_path = '/pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0080/58AAEAC1-BBC8-DE11-93CC-001731A28461.root', drive = 'stkenmvr195a:/dev/rmt/tps0d0n:1310155266', pnfs_id = '0007000000000000622F1F78' where bfid = 'CDMS125731443100000'; 00:00:32 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM 00:00:32 cmsstor206.fnal.gov 001751 root I ENCP MSG_TYPE=ENCP_XFER write_to_hsm /storage/data2/write-pool-2/data/0007000000000000622F1F78 -> /pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0080/58AAEAC1-BBC8-DE11-93CC-001731A28461.root: 2348017013 bytes copied to VOR957 at 85.6 MB/S (103 MB/S network) (112 MB/S drive) (101 MB/S disk) mover=LTO4_95.mover drive_id=ULTRIUM-TD4 drive_sn=1310155266 drive_vendor=IBM elapsed=1633 {'storage_group': 'cms', 'encp_crc': 2230142374L, 'disk_rate': 100.80369599139277, 'drive_rate': 111.95190513298627, 'driver': 'FTTDriver', 'transfer_rate': 85.614949917349179, 'mover_interface': 'stkenmvr195a', 'overall_rate': 83.785479503536806, 'media_changer': 'SL8500G1.media_changer', 'network_rate': 85.614949917349179, 'encp_ip': '131.225.191.77', 'unique_id': 'cmsstor206.fnal.gov-1257312799-1751-0'} 00:00:32 cmsstor206.fnal.gov 001751 root I ENCP File /pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0080/58AAEAC1-BBC8-DE11-93CC-001731A28461.root copied successfully. 00:00:32 cmsstor206.fnal.gov 001751 root I ENCP Completed transferring 2348017013 bytes in 1 files in 1633.04373312 sec. Overall rate = 83.8 MB/sec. Transfer rate = 85.6 MB/sec. Network rate = 103 MB/sec. Drive rate = 112 MB/sec. Disk rate = 101 MB/sec. Exit status = 0. 00:00:32 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:32 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM write_to_hsm next work on vol=VOR957 mover=LTO4_95.mover requester:root 00:00:32 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM HAVE_BOUND:sending write_to_hsm /pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v1/0084/802D2DBC-D3C8-DE11-9DFD-001731AF68B7.root to mover LTO4_95.mover ('131.225.13.190', 7856) DEL_DISM None 00:00:32 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM MSG_TYPE=ADD_TO_LMQ request added to work at movers queue for storage group : cms 00:00:32 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM HAVE_BOUND:sending {'vc': {'storage_group': 'cms', 'external_label': 'VOR957', 'library': 'CD-LTO4G1', 'file_family': 'cms11', 'wrapper': 'cpio_odc', 'address': ('131.225.13.59', 7502), 'volume_family': 'cms.cms11.cpio_odc', 'file_family_width': 10}, 'outfilepath': '/pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v1/0084/802D2DBC-D3C8-DE11-9DFD-001731AF68B7.root', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 25, 'delayed_dismount': None, 'curpri': 25}, 'file_size': 3226538961L, 'mover': 'LTO4_95.mover', 'ignore_fair_share': None, 'retry': None, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v1/0084/802D2DBC-D3C8-DE11-9DFD-001731AF68B7.root', 'uid': 0, 'uname': 'root', 'type': 'cpio_odc', 'gid': 0, 'mtime': 1257312835, 'rmajor': 0, 'machine': ('Linux', 'cmsstor53.fnal.gov', '2.6.31', '#1 SMP Mon Sep 14 09:43:20 CDT 2009', 'x86_64'), 'sanity_size': 65536, 'pstat': (16893, 121762968, 21L, 1, 9811, 5063, 512, 1257298080, 1257298080, 1257293038), 'mode_octal': '0100036', 'mode': 33188, 'gname': 'root', 'size_bytes': 3226538961L, 'fullname': '/storage/data1/write-pool-1/data/00070000000000006242D1D8', 'inode': 121819608L, 'minor': 21}, 'version': 'v3_7d CVS $Revision: 1.922 $ encp', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/storage/data1/write-pool-1/data/00070000000000006242D1D8', 'status': ('ok', None), 'outfile': '/pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v1/0084/.(access)(00070000000000006242D1D8)', 'fc': {'size': 3226538961L, 'external_label': 'VOR957', 'address': ('131.225.13.59', 7501)}, 'infilepath': '/storage/data1/write-pool-1/data/00070000000000006242D1D8', 'at_the_top': 12, 'local_inode': 1073746160, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.204.229', 58402), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'lm_dequeued': 1257314432.1762609, 'encp_start_time': 1257312834.3167911, 'job_queued': 1257312835.57727, 't0': 1257312834, 'in_queue': 1596.5986759662628}, 'user_level': 'admin', 'unique_id': 'cmsstor53.fnal.gov-1257312835-7397-0'} to mover 00:00:32 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:32 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:32 stkenmvr195a.fnal.gov 021479 root I LTO4_95MV WRITE_TO_HSM Thread MainThread 00:00:32 stkenmvr195a.fnal.gov 021479 root I LTO4_95MV SENDING {'vc': {'storage_group': 'cms', 'external_label': 'VOR957', 'library': 'CD-LTO4G1', 'file_family': 'cms11', 'wrapper': 'cpio_odc', 'address': ('131.225.13.59', 7502), 'volume_family': 'cms.cms11.cpio_odc', 'file_family_width': 10}, 'outfilepath': '/pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v1/0084/802D2DBC-D3C8-DE11-9DFD-001731AF68B7.root', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 25, 'delayed_dismount': None, 'curpri': 25}, 'file_size': 3226538961L, 'mover': {'max_buffer': 2621440000L, 'check_written_file': 600, 'max_failures': 5, 'library': 'CD-LTO4G1.library_manager', 'local_mover': 0, 'single_filemark': 1, 'max_consecutive_failures': 3, 'norestart': 'INQ', 'port': 7856, 'mc_device': '0,7,1,14', 'compression': 0, 'serial_num': '1310155266', 'update_interval_in_bound': 2, 'update_interval': 5, 'syslog_entry': 'sense|st[0-9]', 'status': ('ok', None), 'max_rate': 73400320.0, 'vendor_id': 'IBM', 'driver': 'FTTDriver', 'host': 'stkenmvr195a', 'max_time_in_state': 1200, 'device': 'stkenmvr195a:/dev/rmt/tps0d0n', 'name': 'LTO4_95.mover', 'send_stats': 1, 'product_id': 'ULTRIUM-TD4', 'statistics_path': '/tmp/enstore/enstore/LTO4_95.stat', 'mover_address': ('131.225.13.190', 7856), 'mount_delay': 15, 'check_first_written_file': 0, 'callback_addr': ('131.225.13.190', 41186), 'data_ip': 'stkenmvr195a', 'logname': 'LTO4_95MV', 'hostip': '131.225.13.190', 'do_cleaning': 'No', 'media_changer': 'SL8500G1.media_changer'}, 'ignore_fair_share': None, 'retry': None, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v1/0084/802D2DBC-D3C8-DE11-9DFD-001731AF68B7.root', 'uid': 0, 'uname': 'root', 'mtime': 1257312835, 'rmajor': 0, 'machine': ('Linux', 'cmsstor53.fnal.gov', '2.6.31', '#1 SMP Mon Sep 14 09:43:20 CDT 2009', 'x86_64'), 'sanity_size': 65536, 'type': 'cpio_odc', 'pstat': (16893, 121762968, 21L, 1, 9811, 5063, 512, 1257298080, 1257298080, 1257293038), 'mode_octal': '0100036', 'mode': 33188, 'size_bytes': 3226538961L, 'gname': 'root', 'gid': 0, 'fullname': '/storage/data1/write-pool-1/data/00070000000000006242D1D8', 'inode': 121819608L, 'minor': 21}, 'version': 'v3_7d CVS $Revision: 1.922 $ encp', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/storage/data1/write-pool-1/data/00070000000000006242D1D8', 'status': ('ok', None), 'outfile': '/pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v1/0084/.(access)(00070000000000006242D1D8)', 'fc': {'size': 3226538961L, 'external_label': 'VOR957', 'address': ('131.225.13.59', 7501)}, 'infilepath': '/storage/data1/write-pool-1/data/00070000000000006242D1D8', 'at_the_top': 12, 'local_inode': 1073746160, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.204.229', 58402), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'lm_dequeued': 1257314432.1762609, 'encp_start_time': 1257312834.3167911, 'job_queued': 1257312835.57727, 't0': 1257312834, 'in_queue': 1596.5986759662628}, 'user_level': 'admin', 'unique_id': 'cmsstor53.fnal.gov-1257312835-7397-0'} Thread client_connect_thread 00:00:32 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:32 cmsstor53.fnal.gov 007397 root I ENCP Control socket ('131.225.204.229', 58402) is connected to ('131.225.13.190', 41187) for cmsstor53.fnal.gov-1257312835-7397-0. elapsed=1598.080sec 00:00:32 cmsstor53.fnal.gov 007397 root I ENCP Received callback ticket from mover LTO4_95.mover for transfer cmsstor53.fnal.gov-1257312835-7397-0. elapsed=1598.090sec 00:00:32 cmsstor53.fnal.gov 007397 root I ENCP Atempting to connect data socket to mover at ('131.225.13.190', 41186). 00:00:32 cmsstor53.fnal.gov 007397 root I ENCP Connecting data socket to mover (('131.225.13.190', 41186)) with interface 131.225.204.229. elapsed=1598.090sec 00:00:32 cmsstor53.fnal.gov 007397 root I ENCP Data socket ('131.225.204.229', 47517) is connected to ('131.225.13.190', 41186). elapsed=1598.100sec 00:00:32 stkenmvr195a.fnal.gov 021479 root I LTO4_95MV tape position: target 172 current 172 0 Thread seek_thread 00:00:32 stkenmvr195a.fnal.gov 021479 root I LTO4_95MV write_tape starting, bytes_to_write=3226539520 Thread tape_thread 00:00:32 cmsstor53.fnal.gov 007397 root I ENCP Opening local file. 00:00:32 stkenmvr195a.fnal.gov 021479 root I LTO4_95MV Write starting. Tape VOR957 absolute location in blocks 3953262 Thread tape_thread 00:00:32 cmsstor53.fnal.gov 007397 root I ENCP pf_status = ('ok', None) skip_layer_cleanup = False 00:00:32 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:32 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:32 stkenmvr207a.fnal.gov 006260 root I LTO4_107MV filemarks written. Tape VOS312 absolute location in blocks 1117898 Thread tape_thread 00:00:32 stkenmvr207a.fnal.gov 006260 root I LTO4_107MV write_tape timing:pull 0.513707876205 write 11.4521212578 crc_check 18.1924836636 freespace 0.326080560684 block_write 30.4843933582 idle 0.576420068741 Thread tape_thread 00:00:32 stkenmvr207a.fnal.gov 006260 root I LTO4_107MV drive stats after write. Tape VOS312 position 52 block 0 block_size 0 bloc_loc 1117898 tot_blocks 762939000 BOT 0 read_err 0 write_err 0 bytes 3957460480 block_write_tot 30.4843933582 tape_rate 129819230.237 Thread tape_thread 00:00:32 stkenmvr207a.fnal.gov 006260 root I LTO4_107MV remaining bytes info in DB 641339392000 estimated 637381931520 reported from drive 637661184000 position 53 Thread tape_thread 00:00:32 stkenmvr207a.fnal.gov 006260 root I LTO4_107MV new bitfile request {'uid': 0, 'external_label': 'VOS312', 'gid': 0, 'location_cookie': '0000_000000000_0000052', 'complete_crc': 2977142388L, 'sanity_cookie': (65536L, 1868612369L), 'size': 3957459876L} Thread tape_thread 00:00:33 stkenmvr207a.fnal.gov 006260 root I LTO4_107MV set remaining: VOS312 637661184000 0000_000000000_0000053 Thread tape_thread 00:00:33 last message repeated 1 times 00:00:33 stkensrv0n.fnal.gov 028762 enstore I VOLSRV Updating update volume set last_access = '2009-11-04 00:00:33', eod_cookie = '0000_000000000_0000053', non_del_files = '52', remaining_bytes = 637661184000 where label = 'VOS312'; 00:00:33 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:33 stkenmvr207a.fnal.gov 006260 root I LTO4_107MV transfer complete volume=VOS312 location=53 Thread tape_thread 00:00:33 last message repeated 1 times 00:00:33 stkensrv0n.fnal.gov 028762 enstore I VOLSRV Updating update volume set sum_wr_access = '52', modification_time = '2009-11-04 00:00:33' where label = 'VOS312'; 00:00:33 stkenmvr207a.fnal.gov 006260 root I LTO4_107MV Sending done to client: {'vc': {'comment': '', 'storage_group': 'cms', 'blocksize': 131072, 'declared': 1255359569.0, 'library': 'CD-LTO4G1', 'si_time': [0.0, 1250997631.0], 'user_inhibit': ['none', 'none'], 'system_inhibit': ['none', 'none'], 'external_label': 'VOS312', 'wrapper': 'cpio_odc', 'remaining_bytes': 641339392000L, 'sum_mounts': 58, 'sum_rd_access': 0, 'file_family_width': 10, 'last_access': 1257314383.0, 'media_type': 'LTO4', 'status': ('ok', None), 'eod_cookie': '0000_000000000_0000052', 'capacity_bytes': 858993459200L, 'non_del_files': 51, 'sum_wr_err': 0, 'sum_wr_access': 51, 'file_family': 'cms11', 'address': ('131.225.13.59', 7502), 'volume_family': 'cms.cms11.cpio_odc', 'modification_time': 1257314383.0, 'write_protected': 'n', 'sum_rd_err': 0, 'first_access': 1257312304.0}, 'outfilepath': '/pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0086/E2629D85-EEC8-DE11-8329-00261894396D.root', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 25, 'delayed_dismount': None, 'curpri': 25}, 'file_size': 3957459876L, 'mover': {'max_buffer': 2621440000L, 'check_written_file': 600, 'max_failures': 5, 'library': 'CD-LTO4G1.library_manager', 'local_mover': 0, 'single_filemark': 1, 'max_consecutive_failures': 3, 'norestart': 'INQ', 'port': 7867, 'mc_device': '0,6,1,4', 'compression': 0, 'serial_num': '9310109861', 'update_interval_in_bound': 2, 'update_interval': 5, 'syslog_entry': 'sense|st[0-9]', 'status': ('ok', None), 'max_rate': 73400320.0, 'vendor_id': 'IBM', 'driver': 'FTTDriver', 'host': 'stkenmvr207a', 'max_time_in_state': 1200, 'device': 'stkenmvr207a:/dev/rmt/tps0d0n', 'name': 'LTO4_107.mover', 'send_stats': 1, 'product_id': 'ULTRIUM-TD4', 'statistics_path': '/tmp/enstore/enstore/LTO4_107.stat', 'mover_address': ('131.225.13.203', 7867), 'mount_delay': 15, 'check_first_written_file': 0, 'callback_addr': ('131.225.13.203', 38326), 'data_ip': 'stkenmvr207a', 'logname': 'LTO4_107MV', 'hostip': '131.225.13.203', 'do_cleaning': 'No', 'media_changer': 'SL8500G1.media_changer'}, 'ignore_fair_share': None, 'retry': None, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0086/E2629D85-EEC8-DE11-8329-00261894396D.root', 'uid': 0, 'uname': 'root', 'mtime': 1257312759, 'rmajor': 0, 'machine': ('Linux', 'cmsstor118.fnal.gov', '2.6.31', '#1 SMP Mon Sep 14 09:43:20 CDT 2009', 'x86_64'), 'sanity_size': 65536, 'type': 'cpio_odc', 'pstat': (16893, 122462792, 21L, 1, 9811, 5063, 512, 1257306551, 1257306551, 1257299476), 'mode_octal': '0100036', 'mode': 33188, 'size_bytes': 3957459876L, 'gname': 'root', 'gid': 0, 'fullname': '/storage/data1/write-pool-1/data/000700000000000062535DD0', 'inode': 122904016L, 'minor': 21}, 'version': 'v3_7d CVS $Revision: 1.922 $ encp', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/storage/data1/write-pool-1/data/000700000000000062535DD0', 'status': ('ok', None), 'outfile': '/pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0086/.(access)(000700000000000062535DD0)', 'fc': {'uid': 0, 'external_label': 'VOS312', 'gid': 0, 'location_cookie': '0000_000000000_0000052', 'complete_crc': 2977142388L, 'bfid': 'CDMS125731443200000', 'sanity_cookie': (65536L, 1868612369L), 'size': 3957459876L}, 'infilepath': '/storage/data1/write-pool-1/data/000700000000000062535DD0', 'at_the_top': 10, 'local_inode': 4112, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.189.61', 60844), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'encp_start_time': 1257312758.1908851, 'seek_time': 0.029979944229125977, 'transfer_time': 49.39852499961853, 't0': 1257312758, 'mount_time': 0.014399051666259766, 'lm_dequeued': 1257314383.7385659, 'job_queued': 1257312759.3715479, 'drive_call_time': 11.029380559921265, 'in_queue': 1624.3666970729828, 'drive_transfer_time': 33.049045324325562}, 'user_level': 'admin', 'unique_id': 'cmsstor118.fnal.gov-1257312759-13119-0'} Thread tape_thread 00:00:33 stkenmvr207a.fnal.gov 006260 root I LTO4_107MV Done is sent Thread tape_thread 00:00:33 stkenmvr207a.fnal.gov 006260 root I LTO4_107MV maybe_clean Thread tape_thread 00:00:33 stkenmvr207a.fnal.gov 006260 root I LTO4_107MV returned from maybe_clean Thread tape_thread 00:00:33 cmsstor118.fnal.gov 013119 root I ENCP Received final dialog for cmsstor118.fnal.gov-1257312759-13119-0. elapsed=1675.260sec 00:00:33 cmsstor118.fnal.gov 013119 root I ENCP leaving transfer_file(). 00:00:33 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:33 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM write_to_hsm next work on vol=VOS312 mover=LTO4_107.mover requester:root 00:00:33 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM HAVE_BOUND:sending write_to_hsm /pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0087/E012DBC4-F0C8-DE11-9E5E-001A92971B8E.root to mover LTO4_107.mover ('131.225.13.203', 7867) DEL_DISM None 00:00:33 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM MSG_TYPE=ADD_TO_LMQ request added to work at movers queue for storage group : cms 00:00:33 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM HAVE_BOUND:sending {'vc': {'storage_group': 'cms', 'external_label': 'VOS312', 'library': 'CD-LTO4G1', 'file_family': 'cms11', 'wrapper': 'cpio_odc', 'address': ('131.225.13.59', 7502), 'volume_family': 'cms.cms11.cpio_odc', 'file_family_width': 10}, 'outfilepath': '/pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0087/E012DBC4-F0C8-DE11-9E5E-001A92971B8E.root', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 25, 'delayed_dismount': None, 'curpri': 25}, 'file_size': 1593666715L, 'mover': 'LTO4_107.mover', 'ignore_fair_share': None, 'retry': None, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0087/E012DBC4-F0C8-DE11-9E5E-001A92971B8E.root', 'uid': 0, 'uname': 'root', 'type': 'cpio_odc', 'gid': 0, 'mtime': 1257312835, 'rmajor': 0, 'machine': ('Linux', 'cmsstor199.fnal.gov', '2.6.31', '#1 SMP Mon Sep 14 09:43:20 CDT 2009', 'x86_64'), 'sanity_size': 65536, 'pstat': (16893, 122945816, 21L, 1, 9811, 5063, 512, 1257307721, 1257307721, 1257304542), 'mode_octal': '0100036', 'mode': 33188, 'gname': 'root', 'size_bytes': 1593666715L, 'fullname': '/storage/data2/write-pool-2/data/00070000000000006254D150', 'inode': 122999120L, 'minor': 21}, 'version': 'v3_7d CVS $Revision: 1.922 $ encp', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/storage/data2/write-pool-2/data/00070000000000006254D150', 'status': ('ok', None), 'outfile': '/pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0087/.(access)(00070000000000006254D150)', 'fc': {'size': 1593666715L, 'external_label': 'VOS312', 'address': ('131.225.13.59', 7501)}, 'infilepath': '/storage/data2/write-pool-2/data/00070000000000006254D150', 'at_the_top': 8, 'local_inode': 7137, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.207.24', 46088), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'lm_dequeued': 1257314433.7877951, 'encp_start_time': 1257312834.8341379, 'job_queued': 1257312835.8573251, 't0': 1257312834, 'in_queue': 1597.9299318790436}, 'user_level': 'admin', 'unique_id': 'cmsstor199.fnal.gov-1257312835-1081-0'} to mover 00:00:33 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:33 stkenmvr207a.fnal.gov 006260 root I LTO4_107MV WRITE_TO_HSM Thread MainThread 00:00:33 stkenmvr207a.fnal.gov 006260 root I LTO4_107MV SENDING {'vc': {'storage_group': 'cms', 'external_label': 'VOS312', 'library': 'CD-LTO4G1', 'file_family': 'cms11', 'wrapper': 'cpio_odc', 'address': ('131.225.13.59', 7502), 'volume_family': 'cms.cms11.cpio_odc', 'file_family_width': 10}, 'outfilepath': '/pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0087/E012DBC4-F0C8-DE11-9E5E-001A92971B8E.root', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 25, 'delayed_dismount': None, 'curpri': 25}, 'file_size': 1593666715L, 'mover': {'max_buffer': 2621440000L, 'check_written_file': 600, 'max_failures': 5, 'library': 'CD-LTO4G1.library_manager', 'local_mover': 0, 'single_filemark': 1, 'max_consecutive_failures': 3, 'norestart': 'INQ', 'port': 7867, 'mc_device': '0,6,1,4', 'compression': 0, 'serial_num': '9310109861', 'update_interval_in_bound': 2, 'update_interval': 5, 'syslog_entry': 'sense|st[0-9]', 'status': ('ok', None), 'max_rate': 73400320.0, 'vendor_id': 'IBM', 'driver': 'FTTDriver', 'host': 'stkenmvr207a', 'max_time_in_state': 1200, 'device': 'stkenmvr207a:/dev/rmt/tps0d0n', 'name': 'LTO4_107.mover', 'send_stats': 1, 'product_id': 'ULTRIUM-TD4', 'statistics_path': '/tmp/enstore/enstore/LTO4_107.stat', 'mover_address': ('131.225.13.203', 7867), 'mount_delay': 15, 'check_first_written_file': 0, 'callback_addr': ('131.225.13.203', 38333), 'data_ip': 'stkenmvr207a', 'logname': 'LTO4_107MV', 'hostip': '131.225.13.203', 'do_cleaning': 'No', 'media_changer': 'SL8500G1.media_changer'}, 'ignore_fair_share': None, 'retry': None, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0087/E012DBC4-F0C8-DE11-9E5E-001A92971B8E.root', 'uid': 0, 'uname': 'root', 'mtime': 1257312835, 'rmajor': 0, 'machine': ('Linux', 'cmsstor199.fnal.gov', '2.6.31', '#1 SMP Mon Sep 14 09:43:20 CDT 2009', 'x86_64'), 'sanity_size': 65536, 'type': 'cpio_odc', 'pstat': (16893, 122945816, 21L, 1, 9811, 5063, 512, 1257307721, 1257307721, 1257304542), 'mode_octal': '0100036', 'mode': 33188, 'size_bytes': 1593666715L, 'gname': 'root', 'gid': 0, 'fullname': '/storage/data2/write-pool-2/data/00070000000000006254D150', 'inode': 122999120L, 'minor': 21}, 'version': 'v3_7d CVS $Revision: 1.922 $ encp', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/storage/data2/write-pool-2/data/00070000000000006254D150', 'status': ('ok', None), 'outfile': '/pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0087/.(access)(00070000000000006254D150)', 'fc': {'size': 1593666715L, 'external_label': 'VOS312', 'address': ('131.225.13.59', 7501)}, 'infilepath': '/storage/data2/write-pool-2/data/00070000000000006254D150', 'at_the_top': 8, 'local_inode': 7137, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.207.24', 46088), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'lm_dequeued': 1257314433.7877951, 'encp_start_time': 1257312834.8341379, 'job_queued': 1257312835.8573251, 't0': 1257312834, 'in_queue': 1597.9299318790436}, 'user_level': 'admin', 'unique_id': 'cmsstor199.fnal.gov-1257312835-1081-0'} Thread client_connect_thread 00:00:33 cmsstor199.fnal.gov 001081 root I ENCP Control socket ('131.225.207.24', 46088) is connected to ('131.225.13.203', 38334) for cmsstor199.fnal.gov-1257312835-1081-0. elapsed=1599.010sec 00:00:33 cmsstor199.fnal.gov 001081 root I ENCP Received callback ticket from mover LTO4_107.mover for transfer cmsstor199.fnal.gov-1257312835-1081-0. elapsed=1599.010sec 00:00:33 cmsstor199.fnal.gov 001081 root I ENCP Atempting to connect data socket to mover at ('131.225.13.203', 38333). 00:00:33 cmsstor199.fnal.gov 001081 root I ENCP Connecting data socket to mover (('131.225.13.203', 38333)) with interface 131.225.207.24. elapsed=1599.010sec 00:00:33 cmsstor199.fnal.gov 001081 root I ENCP Data socket ('131.225.207.24', 40602) is connected to ('131.225.13.203', 38333). elapsed=1599.010sec 00:00:33 cmsstor199.fnal.gov 001081 root I ENCP Opening local file. 00:00:33 stkenmvr207a.fnal.gov 006260 root I LTO4_107MV tape position: target 53 current 53 0 Thread seek_thread 00:00:33 stkenmvr207a.fnal.gov 006260 root I LTO4_107MV write_tape starting, bytes_to_write=1593667072 Thread tape_thread 00:00:33 cmsstor199.fnal.gov 001081 root I ENCP pf_status = ('ok', None) skip_layer_cleanup = False 00:00:33 stkenmvr207a.fnal.gov 006260 root I LTO4_107MV Write starting. Tape VOS312 absolute location in blocks 1117898 Thread tape_thread 00:00:33 stkendca17a.fnal.gov 025992 root I ENCP File /diskc/write-pool-3/data/0030000000000000061D1078 transfered for stkendca17a.fnal.gov-1257314337-25992-0 in 26.9808719158 seconds. 00:00:34 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM 00:00:34 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:34 cmsstor118.fnal.gov 013119 root I ENCP pf_status = ('ok', None) skip_layer_cleanup = False 00:00:34 stkendca22a.fnal.gov 010240 root I ENCP File /diskb/write-pool-2/data/000F0000000000000B20C088 transfered for stkendca22a.fnal.gov-1257314235-10240-0 in 2.99270105362 seconds. 00:00:34 stkensrv0n.fnal.gov 028796 enstore I FILSRV Updating update file set deleted = 'n', pnfs_path = '/pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0086/E2629D85-EEC8-DE11-8329-00261894396D.root', drive = 'stkenmvr207a:/dev/rmt/tps0d0n:9310109861', pnfs_id = '000700000000000062535DD0' where bfid = 'CDMS125731443200000'; 00:00:34 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:34 stkenmvr197a.fnal.gov 025501 root I LTO4_97MV written bytes 289488384/289488384, blocks 2209 header 180 trailer 282 Thread tape_thread 00:00:34 cmsstor118.fnal.gov 013119 root I ENCP MSG_TYPE=ENCP_XFER write_to_hsm /storage/data1/write-pool-1/data/000700000000000062535DD0 -> /pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0086/E2629D85-EEC8-DE11-8329-00261894396D.root: 3957459876 bytes copied to VOS312 at 76.5 MB/S (90.9 MB/S network) (114 MB/S drive) (96.5 MB/S disk) mover=LTO4_107.mover drive_id=ULTRIUM-TD4 drive_sn=9310109861 drive_vendor=IBM elapsed=1676.5 {'storage_group': 'cms', 'encp_crc': 2977142388L, 'disk_rate': 96.547669211645527, 'drive_rate': 114.19778710435359, 'driver': 'FTTDriver', 'transfer_rate': 76.547763496115621, 'mover_interface': 'stkenmvr207a', 'overall_rate': 74.361478702438134, 'media_changer': 'SL8500G1.media_changer', 'network_rate': 76.547763496115621, 'encp_ip': '131.225.189.61', 'unique_id': 'cmsstor118.fnal.gov-1257312759-13119-0'} 00:00:34 cmsstor118.fnal.gov 013119 root I ENCP File /pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0086/E2629D85-EEC8-DE11-8329-00261894396D.root copied successfully. 00:00:34 cmsstor118.fnal.gov 013119 root I ENCP Completed transferring 3957459876 bytes in 1 files in 1676.45620489 sec. Overall rate = 74.4 MB/sec. Transfer rate = 76.5 MB/sec. Network rate = 90.9 MB/sec. Drive rate = 114 MB/sec. Disk rate = 96.5 MB/sec. Exit status = 0. 00:00:34 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:34 cmsstor206.fnal.gov 020768 root I ENCP Version: v3_7d CVS $Revision: 1.922 $ encp OS: Linux 2.6.31 x86_64 Release: Scientific Linux SL release 5.2 (Boron) Rocks release 5.1 (V.I) User: root(0) Group: root(0) Euser: root(0) Egroup: root(0) Library: CD-LTO4G1 Storage Group: cms File Family: cms11 FF Wrapper: cpio_odc FF Width: 10 Current working directory: cmsstor206.fnal.gov:/ Command line: /home/enstore/enstore/bin/encp --verbose 4 --threaded --ecrc --resubmit-timeout 1800 --priority 25 --pnfs-mount /pnfs/fs --put-cache 0007000000000000622E3160 /storage/data1/write-pool-1/data/0007000000000000622E3160 00:00:34 cmsstor65.fnal.gov 008176 root I ENCP Version: v3_7d CVS $Revision: 1.922 $ encp OS: Linux 2.6.31 x86_64 Release: Scientific Linux SL release 5.2 (Boron) Rocks release 5.1 (V.I) User: root(0) Group: root(0) Euser: root(0) Egroup: root(0) Library: CD-LTO4G1 Storage Group: cms File Family: cms11 FF Wrapper: cpio_odc FF Width: 10 Current working directory: cmsstor65.fnal.gov:/ Command line: /home/enstore/enstore/bin/encp --verbose 4 --threaded --ecrc --resubmit-timeout 1800 --priority 25 --pnfs-mount /pnfs/fs --put-cache 00070000000000006246A0C0 /storage/data2/write-pool-2/data/00070000000000006246A0C0 00:00:35 cmsstor206.fnal.gov 020768 root I ENCP Creating zero length output files. 00:00:35 cmsstor206.fnal.gov 020768 root I ENCP Sending /pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0080/5E13997D-BAC8-DE11-B301-003048678B12.root write request to LM: unique_id: cmsstor206.fnal.gov-1257314435-20768-0 inputfile: /storage/data1/write-pool-1/data/0007000000000000622E3160 outputfile: /pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0080/.(access)(0007000000000000622E3160) 00:00:35 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM MSG_TYPE=ADD_TO_LMQ request added to pending queue for storage group : cms 00:00:35 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM write Q'd /storage/data1/write-pool-1/data/0007000000000000622E3160 (cmsstor206.fnal.gov-1257314435-20768-0) -> /pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0080/5E13997D-BAC8-DE11-B301-003048678B12.root : library=CD-LTO4G1 family=cms11 requester:root volume_family:cms.cms11.cpio_odc 00:00:35 cmsstor206.fnal.gov 020768 root I ENCP Listening for control socket at: ('131.225.191.77', 37870) 00:00:35 cmsstor206.fnal.gov 020768 root I ENCP Waiting for mover to connect control socket at address: ('131.225.191.77', 37870) elapsed=0.950sec 00:00:35 cmsstor65.fnal.gov 008176 root I ENCP Creating zero length output files. 00:00:35 cmsstor65.fnal.gov 008176 root I ENCP Sending /pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0084/F463EC09-DBC8-DE11-804B-001A92810AE4.root write request to LM: unique_id: cmsstor65.fnal.gov-1257314435-8176-0 inputfile: /storage/data2/write-pool-2/data/00070000000000006246A0C0 outputfile: /pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0084/.(access)(00070000000000006246A0C0) 00:00:35 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM MSG_TYPE=ADD_TO_LMQ request added to pending queue for storage group : cms 00:00:35 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM write Q'd /storage/data2/write-pool-2/data/00070000000000006246A0C0 (cmsstor65.fnal.gov-1257314435-8176-0) -> /pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0084/F463EC09-DBC8-DE11-804B-001A92810AE4.root : library=CD-LTO4G1 family=cms11 requester:root volume_family:cms.cms11.cpio_odc 00:00:35 cmsstor65.fnal.gov 008176 root I ENCP Listening for control socket at: ('131.225.204.241', 50860) 00:00:35 cmsstor65.fnal.gov 008176 root I ENCP Waiting for mover to connect control socket at address: ('131.225.204.241', 50860) elapsed=1.670sec 00:00:35 cdfenmvr192a.fnal.gov 006151 root I LTO4_92MV filemarks written. Tape VOV138 absolute location in blocks 255529 Thread tape_thread 00:00:35 cdfenmvr192a.fnal.gov 006151 root I LTO4_92MV write_tape timing:pull 0.00042462348938 write 0.00838232040405 crc_check 0.0318510532379 freespace 0.000413417816162 block_write 0.0410714149475 idle 0.000830173492432 Thread tape_thread 00:00:35 cdfenmvr192a.fnal.gov 006151 root I LTO4_92MV drive stats after write. Tape VOV138 position 5188 block 0 block_size 0 bloc_loc 255529 tot_blocks 762939000 BOT 0 read_err 0 write_err 14 bytes 6970368 block_write_tot 0.0410714149475 tape_rate 169713364.122 Thread tape_thread 00:00:35 cdfenmvr192a.fnal.gov 006151 root I LTO4_92MV remaining bytes info in DB 738913280000 estimated 738906309632 reported from drive 738902016000 position 5191 Thread tape_thread 00:00:35 cdfenmvr192a.fnal.gov 006151 root I LTO4_92MV new bitfile request {'uid': 0, 'external_label': 'VOV138', 'gid': 0, 'location_cookie': '0000_000000000_0005189', 'complete_crc': 1665510970L, 'sanity_cookie': (65536L, 3818063612L), 'size': 6969932L} Thread tape_thread 00:00:35 cdfenmvr192a.fnal.gov 006151 root I LTO4_92MV set remaining: VOV138 738902016000 0000_000000000_0005191 Thread tape_thread 00:00:36 last message repeated 1 times 00:00:36 stkensrv0n.fnal.gov 028762 enstore I VOLSRV Updating update volume set last_access = '2009-11-04 00:00:36', eod_cookie = '0000_000000000_0005191', non_del_files = '1731', remaining_bytes = 738902016000 where label = 'VOV138'; 00:00:36 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM 00:00:36 cdfenmvr192a.fnal.gov 006151 root I LTO4_92MV transfer complete volume=VOV138 location=5191 Thread tape_thread 00:00:36 last message repeated 1 times 00:00:36 stkensrv0n.fnal.gov 028762 enstore I VOLSRV Updating update volume set sum_wr_access = '1730', modification_time = '2009-11-04 00:00:36' where label = 'VOV138'; 00:00:36 cdfenmvr192a.fnal.gov 006151 root I LTO4_92MV Sending done to client: {'vc': {'comment': '', 'storage_group': 'ktev', 'blocksize': 131072, 'declared': 1254165875.0, 'library': 'CD-LTO4F1', 'si_time': [0.0, 0.0], 'user_inhibit': ['none', 'none'], 'system_inhibit': ['none', 'none'], 'external_label': 'VOV138', 'wrapper': 'cern', 'remaining_bytes': 738913280000L, 'sum_mounts': 1, 'sum_rd_access': 0, 'file_family_width': 1, 'last_access': 1257314429.0, 'media_type': 'LTO4', 'status': ('ok', None), 'eod_cookie': '0000_000000000_0005188', 'capacity_bytes': 858993459200L, 'non_del_files': 1730, 'sum_wr_err': 0, 'sum_wr_access': 1729, 'file_family': 'ktev_migration-MIGRATION', 'address': ('131.225.13.59', 7502), 'volume_family': 'ktev.ktev_migration-MIGRATION.cern', 'modification_time': 1257314429.0, 'write_protected': 'n', 'sum_rd_err': 0, 'first_access': 1257302588.0}, 'outfilepath': '/pnfs/fs/usr/Migration/ktev/migrated_from_dlt/KQ/KQRC/KQRC42/f1909', 'encp': {'agetime': 30, 'adminpri': 6, 'delpri': 1, 'basepri': 1000007, 'delayed_dismount': 2, 'curpri': 1000007}, 'file_size': 6969932L, 'mover': {'max_buffer': 2621440000L, 'check_written_file': 600, 'max_failures': 5, 'library': 'CD-LTO4F1.library_manager', 'local_mover': 0, 'single_filemark': 1, 'max_consecutive_failures': 3, 'norestart': 'INQ', 'port': 7853, 'mc_device': '1,0,1,1', 'compression': 0, 'serial_num': '1310060800', 'update_interval_in_bound': 2, 'update_interval': 5, 'syslog_entry': 'sense|st[0-9]', 'status': ('ok', None), 'max_rate': 73400320.0, 'vendor_id': 'IBM', 'driver': 'FTTDriver', 'host': 'cdfenmvr192a', 'max_time_in_state': 1200, 'device': 'cdfenmvr192a:/dev/rmt/tps0d0n', 'name': 'LTO4_92.mover', 'send_stats': 1, 'product_id': 'ULTRIUM-TD4', 'statistics_path': '/tmp/enstore/enstore/LTO4_92.stat', 'mover_address': ('131.225.215.76', 7853), 'mount_delay': 15, 'check_first_written_file': 0, 'callback_addr': ('131.225.215.76', 60660), 'data_ip': 'cdfenmvr192a', 'logname': 'LTO4_92MV', 'hostip': '131.225.215.76', 'do_cleaning': 'No', 'media_changer': 'SL8500F1.media_changer'}, 'ignore_fair_share': 1, 'retry': None, 'lm': {'address': ('131.225.13.4', 8542)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fs/usr/ktev/migrated_from_dlt/KQ/KQRC/KQRC42/f1909', 'uid': 0, 'uname': 'root', 'mtime': 1257314417, 'rmajor': 0, 'machine': ('Linux', 'stkendm7a.fnal.gov', '2.6.9-89.0.9.ELsmp', '#1 SMP Mon Aug 24 13:47:43 CDT 2009', 'i686'), 'sanity_size': 65536, 'type': 'cern', 'pstat': (16877, 678574128L, 20L, 1, 0, 0, 512L, 1257314299, 1257314299, 1252942651), 'mode_octal': '0100016', 'mode': 33188, 'size_bytes': 6969932L, 'gname': 'root', 'gid': 0, 'fullname': '/data/data2/Migration_Spool/VO6153:0000_000000000_0032558', 'inode': 675389816L, 'minor': 20}, 'version': 'v3_8 CVS $Revision: 1.963 $ encp.pyc', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/data/data2/Migration_Spool/VO6153:0000_000000000_0032558', 'status': ('ok', None), 'outfile': '/pnfs/fs/usr/Migration/ktev/migrated_from_dlt/KQ/KQRC/KQRC42/.(access)(00280000000000000841A178)', 'fc': {'uid': 0, 'external_label': 'VOV138', 'gid': 0, 'location_cookie': '0000_000000000_0005189', 'complete_crc': 1665510970L, 'bfid': 'CDMS125731443500000', 'sanity_cookie': (65536L, 3818063612L), 'size': 6969932L}, 'infilepath': '/data/data2/Migration_Spool/VO6153:0000_000000000_0032558', 'at_the_top': 39, 'local_inode': 4495421L, 'resend': {'max_retry': 3, 'resubmits': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.13.19', 57448), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'encp_start_time': 1257314417.5619681, 'seek_time': 0.026795148849487305, 'transfer_time': 5.4994339942932129, 't0': 1257314417, 'mount_time': 0.057606935501098633, 'lm_dequeued': 1257314430.407619, 'job_queued': 1257314417.7896371, 'drive_call_time': 0.018730401992797852, 'in_queue': 12.617725849151611, 'drive_transfer_time': 5.3402810096740723}, 'user_level': 'admin', 'unique_id': 'stkendm7a.fnal.gov-1257314417-18643-24634'} Thread tape_thread 00:00:36 cdfenmvr192a.fnal.gov 006151 root I LTO4_92MV Done is sent Thread tape_thread 00:00:36 cdfenmvr192a.fnal.gov 006151 root I LTO4_92MV maybe_clean Thread tape_thread 00:00:36 stkendm7a.fnal.gov 018643 root I ENCP Received final dialog for stkendm7a.fnal.gov-1257314417-18643-24634. elapsed=154682.480sec Thread WRITE_1 00:00:36 stkendm7a.fnal.gov 018643 root I ENCP leaving transfer_file(). Thread WRITE_1 00:00:36 cdfenmvr192a.fnal.gov 006151 root I LTO4_92MV returned from maybe_clean Thread tape_thread 00:00:36 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:36 stkensrv0n.fnal.gov 028796 enstore I FILSRV Updating update file set deleted = 'n', pnfs_path = '/pnfs/fs/usr/ktev/migrated_from_dlt/KQ/KQRC/KQRC42/f1909', drive = 'cdfenmvr192a:/dev/rmt/tps0d0n:1310060800', pnfs_id = '00280000000000000841A178' where bfid = 'CDMS125731443500000'; 00:00:36 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM 00:00:36 stkendm7a.fnal.gov 018643 root I ENCP MSG_TYPE=ENCP_XFER write_to_hsm /data/data2/Migration_Spool/VO6153:0000_000000000_0032558 -> /pnfs/fs/usr/Migration/ktev/migrated_from_dlt/KQ/KQRC/KQRC42/f1909: 6969932 bytes copied to VOV138 at 1.21 MB/S (93.4 MB/S network) (1.24 MB/S drive) (81.6 MB/S disk) mover=LTO4_92.mover drive_id=ULTRIUM-TD4 drive_sn=1310060800 drive_vendor=IBM elapsed=18.838 {'storage_group': 'ktev', 'encp_crc': 1665510970L, 'disk_rate': 81.581859440559654, 'drive_rate': 1.2446995061602066, 'driver': 'FTTDriver', 'transfer_rate': 1.2076634879567745, 'mover_interface': 'cdfenmvr192a', 'overall_rate': 1.1155064146445814, 'media_changer': 'SL8500F1.media_changer', 'network_rate': 1.2076634879567745, 'encp_ip': '131.225.13.19', 'unique_id': 'stkendm7a.fnal.gov-1257314417-18643-24634'} Thread WRITE_1 00:00:36 stkendm7a.fnal.gov 018643 root I ENCP File /pnfs/fs/usr/Migration/ktev/migrated_from_dlt/KQ/KQRC/KQRC42/f1909 copied successfully. Thread WRITE_1 00:00:36 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM write_to_hsm next work on vol=VOV138 mover=LTO4_92.mover requester:root 00:00:36 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM HAVE_BOUND:sending write_to_hsm /pnfs/fs/usr/ktev/migrated_from_dlt/KQ/KQRC/KQRC44/f1969 to mover LTO4_92.mover ('131.225.215.76', 7853) DEL_DISM 2 00:00:36 stkendm7a.fnal.gov 018643 root I ENCP Completed transferring 6969932 bytes in 1 files in 18.8394589424 sec. Overall rate = 1.12 MB/sec. Transfer rate = 1.21 MB/sec. Network rate = 93.4 MB/sec. Drive rate = 1.24 MB/sec. Disk rate = 81.6 MB/sec. Exit status = 0. Thread WRITE_1 00:00:36 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM MSG_TYPE=ADD_TO_LMQ request added to work at movers queue for storage group : ktev 00:00:36 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM HAVE_BOUND:sending {'vc': {'storage_group': 'ktev', 'external_label': 'VOV138', 'library': 'CD-LTO4F1', 'file_family': 'ktev_migration-MIGRATION', 'wrapper': 'cern', 'address': ('131.225.13.59', 7502), 'volume_family': 'ktev.ktev_migration-MIGRATION.cern', 'file_family_width': 1}, 'outfilepath': '/pnfs/fs/usr/Migration/ktev/migrated_from_dlt/KQ/KQRC/KQRC44/f1969', 'encp': {'agetime': 30, 'adminpri': 6, 'delpri': 1, 'basepri': 1000007, 'delayed_dismount': 2, 'curpri': 1000007}, 'file_size': 2304952L, 'mover': 'LTO4_92.mover', 'ignore_fair_share': 1, 'retry': None, 'lm': {'address': ('131.225.13.4', 8542)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fs/usr/ktev/migrated_from_dlt/KQ/KQRC/KQRC44/f1969', 'uid': 0, 'uname': 'root', 'type': 'cern', 'gid': 0, 'mtime': 1257314424, 'rmajor': 0, 'machine': ('Linux', 'stkendm7a.fnal.gov', '2.6.9-89.0.9.ELsmp', '#1 SMP Mon Aug 24 13:47:43 CDT 2009', 'i686'), 'sanity_size': 65536, 'pstat': (16877, 678574824L, 20L, 1, 0, 0, 512L, 1257314410, 1257314410, 1252942696), 'mode_octal': '0100016', 'mode': 33188, 'gname': 'root', 'size_bytes': 2304952L, 'fullname': '/data/data2/Migration_Spool/VO6153:0000_000000000_0032309', 'inode': 675389848L, 'minor': 20}, 'version': 'v3_8 CVS $Revision: 1.963 $ encp.pyc', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/data/data2/Migration_Spool/VO6153:0000_000000000_0032309', 'status': ('ok', None), 'outfile': '/pnfs/fs/usr/Migration/ktev/migrated_from_dlt/KQ/KQRC/KQRC44/.(access)(00280000000000000841A198)', 'fc': {'size': 2304952L, 'external_label': 'VOV138', 'pnfsid': '00280000000000000841A198', 'address': ('131.225.13.59', 7501)}, 'infilepath': '/data/data2/Migration_Spool/VO6153:0000_000000000_0032309', 'at_the_top': 39, 'local_inode': 4495350L, 'resend': {'max_retry': 3, 'resubmits': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.13.19', 57451), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'lm_dequeued': 1257314436.4035311, 'encp_start_time': 1257314424.4886069, 'job_queued': 1257314424.7389269, 't0': 1257314424, 'in_queue': 11.664335012435913}, 'user_level': 'admin', 'unique_id': 'stkendm7a.fnal.gov-1257314424-18643-24635'} to mover 00:00:36 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM 00:00:36 cdfenmvr192a.fnal.gov 006151 root I LTO4_92MV WRITE_TO_HSM Thread MainThread 00:00:36 stkendm7a.fnal.gov 018643 root I ENCP Listening for control socket at: ('131.225.13.19', 57451) Thread WRITE_2 00:00:36 cdfenmvr192a.fnal.gov 006151 root I LTO4_92MV SENDING {'vc': {'storage_group': 'ktev', 'external_label': 'VOV138', 'library': 'CD-LTO4F1', 'file_family': 'ktev_migration-MIGRATION', 'wrapper': 'cern', 'address': ('131.225.13.59', 7502), 'volume_family': 'ktev.ktev_migration-MIGRATION.cern', 'file_family_width': 1}, 'outfilepath': '/pnfs/fs/usr/Migration/ktev/migrated_from_dlt/KQ/KQRC/KQRC44/f1969', 'encp': {'agetime': 30, 'adminpri': 6, 'delpri': 1, 'basepri': 1000007, 'delayed_dismount': 2, 'curpri': 1000007}, 'file_size': 2304952L, 'mover': {'max_buffer': 2621440000L, 'check_written_file': 600, 'max_failures': 5, 'library': 'CD-LTO4F1.library_manager', 'local_mover': 0, 'single_filemark': 1, 'max_consecutive_failures': 3, 'norestart': 'INQ', 'port': 7853, 'mc_device': '1,0,1,1', 'compression': 0, 'serial_num': '1310060800', 'update_interval_in_bound': 2, 'update_interval': 5, 'syslog_entry': 'sense|st[0-9]', 'status': ('ok', None), 'max_rate': 73400320.0, 'vendor_id': 'IBM', 'driver': 'FTTDriver', 'host': 'cdfenmvr192a', 'max_time_in_state': 1200, 'device': 'cdfenmvr192a:/dev/rmt/tps0d0n', 'name': 'LTO4_92.mover', 'send_stats': 1, 'product_id': 'ULTRIUM-TD4', 'statistics_path': '/tmp/enstore/enstore/LTO4_92.stat', 'mover_address': ('131.225.215.76', 7853), 'mount_delay': 15, 'check_first_written_file': 0, 'callback_addr': ('131.225.215.76', 60662), 'data_ip': 'cdfenmvr192a', 'logname': 'LTO4_92MV', 'hostip': '131.225.215.76', 'do_cleaning': 'No', 'media_changer': 'SL8500F1.media_changer'}, 'ignore_fair_share': 1, 'retry': None, 'lm': {'address': ('131.225.13.4', 8542)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fs/usr/ktev/migrated_from_dlt/KQ/KQRC/KQRC44/f1969', 'uid': 0, 'uname': 'root', 'mtime': 1257314424, 'rmajor': 0, 'machine': ('Linux', 'stkendm7a.fnal.gov', '2.6.9-89.0.9.ELsmp', '#1 SMP Mon Aug 24 13:47:43 CDT 2009', 'i686'), 'sanity_size': 65536, 'type': 'cern', 'pstat': (16877, 678574824L, 20L, 1, 0, 0, 512L, 1257314410, 1257314410, 1252942696), 'mode_octal': '0100016', 'mode': 33188, 'size_bytes': 2304952L, 'gname': 'root', 'gid': 0, 'fullname': '/data/data2/Migration_Spool/VO6153:0000_000000000_0032309', 'inode': 675389848L, 'minor': 20}, 'version': 'v3_8 CVS $Revision: 1.963 $ encp.pyc', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/data/data2/Migration_Spool/VO6153:0000_000000000_0032309', 'status': ('ok', None), 'outfile': '/pnfs/fs/usr/Migration/ktev/migrated_from_dlt/KQ/KQRC/KQRC44/.(access)(00280000000000000841A198)', 'fc': {'size': 2304952L, 'external_label': 'VOV138', 'pnfsid': '00280000000000000841A198', 'address': ('131.225.13.59', 7501)}, 'infilepath': '/data/data2/Migration_Spool/VO6153:0000_000000000_0032309', 'at_the_top': 39, 'local_inode': 4495350L, 'resend': {'max_retry': 3, 'resubmits': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.13.19', 57451), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'lm_dequeued': 1257314436.4035311, 'encp_start_time': 1257314424.4886069, 'job_queued': 1257314424.7389269, 't0': 1257314424, 'in_queue': 11.664335012435913}, 'user_level': 'admin', 'unique_id': 'stkendm7a.fnal.gov-1257314424-18643-24635'} Thread client_connect_thread 00:00:36 stkendm7a.fnal.gov 018643 root I ENCP Waiting for mover to connect control socket at address: ('131.225.13.19', 57451) elapsed=154682.860sec Thread WRITE_2 00:00:36 stkendm7a.fnal.gov 018643 root I ENCP Control socket ('131.225.13.19', 57451) is connected to ('131.225.215.76', 60663) for stkendm7a.fnal.gov-1257314424-18643-24635. elapsed=154682.860sec Thread WRITE_2 00:00:36 stkendm7a.fnal.gov 018643 root I ENCP Received callback ticket from mover LTO4_92.mover for transfer stkendm7a.fnal.gov-1257314424-18643-24635. elapsed=154682.870sec Thread WRITE_2 00:00:36 stkendm7a.fnal.gov 018643 root I ENCP Attempting to connect data socket to mover at ('131.225.215.76', 60662). Thread WRITE_2 00:00:36 stkendm7a.fnal.gov 018643 root I ENCP Connecting data socket to mover (('131.225.215.76', 60662)) with interface 131.225.13.19. elapsed=154682.870sec Thread WRITE_2 00:00:36 stkendm7a.fnal.gov 018643 root I ENCP Data socket ('131.225.13.19', 57455) is connected to ('131.225.215.76', 60662). elapsed=154682.870sec Thread WRITE_2 00:00:36 stkendm7a.fnal.gov 018643 root I ENCP Opening local file. Thread WRITE_2 00:00:36 stkendm7a.fnal.gov 018643 root I ENCP pf_status = ('ok', None) skip_layer_cleanup = False Thread WRITE_2 00:00:36 cdfenmvr192a.fnal.gov 006151 root I LTO4_92MV tape position: target 5191 current 5191 0 Thread seek_thread 00:00:36 cdfenmvr192a.fnal.gov 006151 root I LTO4_92MV write_tape starting, bytes_to_write=2305024 Thread tape_thread 00:00:36 cdfenmvr192a.fnal.gov 006151 root I LTO4_92MV Write starting. Tape VOV138 absolute location in blocks 255529 Thread tape_thread 00:00:36 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:36 stkendm7a.fnal.gov 018643 root I ENCP File /data/data2/Migration_Spool/VO6153:0000_000000000_0032309 transfered for stkendm7a.fnal.gov-1257314424-18643-24635 in 0.0279731750488 seconds. Thread WRITE_2 00:00:36 stkensrv0n.fnal.gov 028796 enstore I FILSRV start modifying {'pnfsid': '00280000000000000841A178', 'pnfs_name0': '/pnfs/fs/usr/ktev/migrated_from_dlt/KQ/KQRC/KQRC42/f1909', 'deleted': 'no', 'external_label': 'VOV138', 'drive': 'cdfenmvr192a:/dev/rmt/tps0d0n:1310060800', 'update': '2009-11-04 00:00:36.149568', 'gid': 0, 'location_cookie': '0000_000000000_0005189', 'complete_crc': 1665510970L, 'uid': 0, 'bfid': 'CDMS125731443500000', 'sanity_cookie': (65536L, 3818063612L), 'size': 6969932L} 00:00:36 stkensrv0n.fnal.gov 028796 enstore I FILSRV Updating update file set pnfs_id = '001F00000000000000EEBC98' where bfid = 'CDMS125731443500000'; 00:00:36 stkensrv0n.fnal.gov 028796 enstore I FILSRV modified to {'pnfsid': '001F00000000000000EEBC98', 'pnfs_name0': '/pnfs/fs/usr/ktev/migrated_from_dlt/KQ/KQRC/KQRC42/f1909', 'deleted': 'no', 'external_label': 'VOV138', 'drive': 'cdfenmvr192a:/dev/rmt/tps0d0n:1310060800', 'update': '2009-11-04 00:00:36.149568', 'gid': 0, 'location_cookie': '0000_000000000_0005189', 'complete_crc': 1665510970L, 'uid': 0, 'bfid': 'CDMS125731443500000', 'sanity_cookie': (65536L, 3818063612L), 'size': 6969932L} 00:00:36 stkendm7a.fnal.gov 018643 root I ENCP Version: v3_8 CVS $Revision: 1.963 $ encp.pyc OS: Linux 2.6.9-89.0.9.ELsmp i686 Release: Scientific Linux Fermi LTS release 4.4 (Wilson) User: root(0) Group: root(0) Euser: root(0) Egroup: root(0) Library: CD-LTO4F1 Storage Group: ktev File Family: ktev_migration-MIGRATION FF Wrapper: cern FF Width: 1 Current working directory: stkendm7a.fnal.gov:/opt/enstore/src Command line: encp --delayed-dismount 2 --ignore-fair-share --threaded --priority 0 --storage-group ktev --file-family ktev_migration-MIGRATION --file-family-wrapper cern --library CD-LTO4F1 --override-path /pnfs/fs/usr/ktev/migrated_from_dlt/KQ/KQH0/KQH069/f2327 --threaded /data/data2/Migration_Spool/VO6153:0000_000000000_0032567 /pnfs/fs/usr/Migration/ktev/migrated_from_dlt/KQ/KQH0/KQH069/f2327 Thread WRITE_1 00:00:36 stkendm7a.fnal.gov 018643 root I ENCP Creating zero length output files. Thread WRITE_1 00:00:36 stkendm7a.fnal.gov 018643 root I ENCP Sending /pnfs/fs/usr/Migration/ktev/migrated_from_dlt/KQ/KQH0/KQH069/f2327 write request to LM: unique_id: stkendm7a.fnal.gov-1257314436-18643-24637 inputfile: /data/data2/Migration_Spool/VO6153:0000_000000000_0032567 outputfile: /pnfs/fs/usr/Migration/ktev/migrated_from_dlt/KQ/KQH0/KQH069/.(access)(00280000000000000841A1F0) Thread WRITE_1 00:00:36 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM MSG_TYPE=ADD_TO_LMQ request added to pending queue for storage group : ktev 00:00:36 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM write Q'd /data/data2/Migration_Spool/VO6153:0000_000000000_0032567 (stkendm7a.fnal.gov-1257314436-18643-24637) -> /pnfs/fs/usr/ktev/migrated_from_dlt/KQ/KQH0/KQH069/f2327 : library=CD-LTO4F1 family=ktev_migration-MIGRATION requester:root volume_family:ktev.ktev_migration-MIGRATION.cern 00:00:37 stkenmvr193a.fnal.gov 031647 root I LTO4_93MV written bytes 1966080512/1966080512, blocks 15001 header 189 trailer 323 Thread tape_thread 00:00:37 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM 00:00:37 stkenmvr210a.fnal.gov 011436 root I LTO4_110MV tape position: target 1829 current 0 1 Thread seek_thread 00:00:37 cmsstor118.fnal.gov 007624 root I ENCP Version: v3_7d CVS $Revision: 1.922 $ encp OS: Linux 2.6.31 x86_64 Release: Rocks release 5.1 (V.I) Scientific Linux SL release 5.2 (Boron) User: root(0) Group: root(0) Euser: root(0) Egroup: root(0) Library: CD-LTO4G1 Storage Group: cms File Family: cms11 FF Wrapper: cpio_odc FF Width: 10 Current working directory: cmsstor118.fnal.gov:/root Command line: /home/enstore/enstore/bin/encp --verbose 4 --threaded --ecrc --resubmit-timeout 1800 --priority 25 --pnfs-mount /pnfs/fs --put-cache 0007000000000000623CDDF8 /storage/data2/write-pool-2/data/0007000000000000623CDDF8 00:00:37 cdfenmvr192a.fnal.gov 006151 root I LTO4_92MV written bytes 2305024/2305024, blocks 18 header 0 trailer 72 Thread tape_thread 00:00:38 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM 00:00:38 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:38 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:38 cmsstor118.fnal.gov 007624 root I ENCP Creating zero length output files. 00:00:38 cmsstor118.fnal.gov 007624 root I ENCP Sending /pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0082/CA62D1D4-CEC8-DE11-A8CC-00248C0BE01E.root write request to LM: unique_id: cmsstor118.fnal.gov-1257314438-7624-0 inputfile: /storage/data2/write-pool-2/data/0007000000000000623CDDF8 outputfile: /pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0082/.(access)(0007000000000000623CDDF8) 00:00:38 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM MSG_TYPE=ADD_TO_LMQ request added to pending queue for storage group : cms 00:00:38 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM write Q'd /storage/data2/write-pool-2/data/0007000000000000623CDDF8 (cmsstor118.fnal.gov-1257314438-7624-0) -> /pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0082/CA62D1D4-CEC8-DE11-A8CC-00248C0BE01E.root : library=CD-LTO4G1 family=cms11 requester:root volume_family:cms.cms11.cpio_odc 00:00:38 cmsstor118.fnal.gov 007624 root I ENCP Listening for control socket at: ('131.225.189.61', 49336) 00:00:38 cmsstor118.fnal.gov 007624 root I ENCP Waiting for mover to connect control socket at address: ('131.225.189.61', 49336) elapsed=1.340sec 00:00:38 cmsstor147.fnal.gov 002565 root I ENCP File /storage/data2/write-pool-2/data/0007000000000000623CD5B0 transfered for cmsstor147.fnal.gov-1257312802-2565-0 in 20.6383290291 seconds. 00:00:38 cmsstor74.fnal.gov 007757 root I ENCP File /storage/data1/write-pool-1/data/00070000000000006257CD10 transfered for cmsstor74.fnal.gov-1257312754-7757-0 in 67.8679559231 seconds. 00:00:38 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:38 stkenmvr193a.fnal.gov 031647 root I LTO4_93MV filemarks written. Tape VON732 absolute location in blocks 3400015 Thread tape_thread 00:00:38 stkenmvr193a.fnal.gov 031647 root I LTO4_93MV write_tape timing:pull 0.197160243988 write 7.84540057182 crc_check 9.12704348564 freespace 0.168026924133 block_write 17.3376312256 idle 0.293413877487 Thread tape_thread 00:00:38 stkenmvr193a.fnal.gov 031647 root I LTO4_93MV drive stats after write. Tape VON732 position 235 block 0 block_size 0 bloc_loc 3400015 tot_blocks 762939000 BOT 0 read_err 0 write_err 0 bytes 1966080512 block_write_tot 17.3376312256 tape_rate 113399603.811 Thread tape_thread 00:00:38 cmsstor133.fnal.gov 002545 root I ENCP Version: v3_7d CVS $Revision: 1.922 $ encp OS: Linux 2.6.31 x86_64 Release: Rocks release 5.1 (V.I) Scientific Linux SL release 5.2 (Boron) User: root(0) Group: root(0) Euser: root(0) Egroup: root(0) Library: CD-LTO4G1 Storage Group: cms File Family: cms11 FF Wrapper: cpio_odc FF Width: 10 Current working directory: cmsstor133.fnal.gov:/root Command line: /home/enstore/enstore/bin/encp --verbose 4 --threaded --ecrc --resubmit-timeout 1800 --priority 25 --pnfs-mount /pnfs/fs --put-cache 000700000000000062546FB0 /storage/data1/write-pool-1/data/000700000000000062546FB0 00:00:38 stkenmvr193a.fnal.gov 031647 root I LTO4_93MV remaining bytes info in DB 361459712000 estimated 359493631488 reported from drive 357943296000 position 236 Thread tape_thread 00:00:38 stkenmvr193a.fnal.gov 031647 root I LTO4_93MV new bitfile request {'uid': 0, 'external_label': 'VON732', 'gid': 0, 'location_cookie': '0000_000000000_0000235', 'complete_crc': 3747111452L, 'sanity_cookie': (65536L, 249140578L), 'size': 1966080000L} Thread tape_thread 00:00:38 stkenmvr193a.fnal.gov 031647 root I LTO4_93MV set remaining: VON732 357943296000 0000_000000000_0000236 Thread tape_thread 00:00:38 last message repeated 1 times 00:00:38 stkensrv0n.fnal.gov 028762 enstore I VOLSRV Updating update volume set last_access = '2009-11-04 00:00:38', eod_cookie = '0000_000000000_0000236', non_del_files = '240', remaining_bytes = 357943296000 where label = 'VON732'; 00:00:38 stkenmvr193a.fnal.gov 031647 root I LTO4_93MV transfer complete volume=VON732 location=236 Thread tape_thread 00:00:38 last message repeated 1 times 00:00:38 stkensrv0n.fnal.gov 028762 enstore I VOLSRV Updating update volume set sum_wr_access = '235', modification_time = '2009-11-04 00:00:38' where label = 'VON732'; 00:00:38 stkenmvr193a.fnal.gov 031647 root I LTO4_93MV Sending done to client: {'vc': {'comment': '', 'storage_group': 'exp-db', 'blocksize': 131072, 'declared': 1256853170.0, 'library': 'CD-LTO4G1', 'si_time': [0.0, 1255427870.0], 'user_inhibit': ['none', 'none'], 'system_inhibit': ['none', 'none'], 'external_label': 'VON732', 'wrapper': 'cpio_odc', 'remaining_bytes': 361459712000L, 'sum_mounts': 45, 'sum_rd_access': 0, 'file_family_width': 2, 'last_access': 1257314406.0, 'media_type': 'LTO4', 'status': ('ok', None), 'eod_cookie': '0000_000000000_0000235', 'capacity_bytes': 858993459200L, 'non_del_files': 239, 'sum_wr_err': 0, 'sum_wr_access': 234, 'file_family': 'daily-d0-offline', 'address': ('131.225.13.59', 7502), 'volume_family': 'exp-db.daily-d0-offline.cpio_odc', 'modification_time': 1257314406.0, 'write_protected': 'n', 'sum_rd_err': 0, 'first_access': 1257262991.0}, 'outfilepath': '/pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/03/channel02/D0OFPRD1_t701979310_s103226_p22', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 1, 'delayed_dismount': None, 'curpri': 1}, 'file_size': 1966080000L, 'mover': {'max_buffer': 2621440000L, 'check_written_file': 600, 'max_failures': 5, 'library': 'CD-LTO4G1.library_manager', 'local_mover': 0, 'single_filemark': 1, 'max_consecutive_failures': 3, 'norestart': 'INQ', 'port': 7854, 'mc_device': '0,7,1,11', 'compression': 0, 'serial_num': '9310097076', 'update_interval_in_bound': 2, 'update_interval': 5, 'syslog_entry': 'sense|st[0-9]', 'status': ('ok', None), 'max_rate': 73400320.0, 'vendor_id': 'IBM', 'driver': 'FTTDriver', 'host': 'stkenmvr193a', 'max_time_in_state': 1200, 'device': 'stkenmvr193a:/dev/rmt/tps0d0n', 'name': 'LTO4_93.mover', 'send_stats': 1, 'product_id': 'ULTRIUM-TD4', 'statistics_path': '/tmp/enstore/enstore/LTO4_93.stat', 'mover_address': ('131.225.13.188', 7854), 'mount_delay': 15, 'check_first_written_file': 0, 'callback_addr': ('131.225.13.188', 36320), 'data_ip': 'stkenmvr193a', 'logname': 'LTO4_93MV', 'hostip': '131.225.13.188', 'do_cleaning': 'No', 'media_changer': 'SL8500G1.media_changer'}, 'ignore_fair_share': None, 'retry': None, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/03/channel02/D0OFPRD1_t701979310_s103226_p22', 'uid': 0, 'uname': 'root', 'mtime': 1257314337, 'rmajor': 0, 'machine': ('Linux', 'stkendca17a.fnal.gov', '2.6.9-89.0.9.ELsmp', '#1 SMP Mon Aug 24 13:47:43 CDT 2009', 'i686'), 'sanity_size': 65536, 'type': 'cpio_odc', 'pstat': (16877, 820618368L, 20L, 1, 1602, 2752, 512L, 1257314296, 1257314296, 1254398418), 'mode_octal': '0100016', 'mode': 33188, 'size_bytes': 1966080000L, 'gname': 'root', 'gid': 0, 'fullname': '/diskc/write-pool-3/data/0030000000000000061D1078', 'inode': 807211128L, 'minor': 20}, 'version': 'v3_8b CVS $Revision: 1.922 $ ', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/diskc/write-pool-3/data/0030000000000000061D1078', 'status': ('ok', None), 'outfile': '/pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/03/channel02/.(access)(0030000000000000061D1078)', 'fc': {'uid': 0, 'external_label': 'VON732', 'gid': 0, 'location_cookie': '0000_000000000_0000235', 'complete_crc': 3747111452L, 'bfid': 'CDMS125731443800000', 'sanity_cookie': (65536L, 249140578L), 'size': 1966080000L}, 'infilepath': '/diskc/write-pool-3/data/0030000000000000061D1078', 'at_the_top': 138, 'local_inode': 268645760L, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.13.5', 39502), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'encp_start_time': 1257314336.353235, 'seek_time': 0.030972957611083984, 'transfer_time': 31.988306045532227, 't0': 1257314336, 'mount_time': 0.016953945159912109, 'lm_dequeued': 1257314406.824646, 'job_queued': 1257314337.2924671, 'drive_call_time': 7.6341369152069092, 'in_queue': 69.532097816467285, 'drive_transfer_time': 19.359724283218384}, 'user_level': 'user2', 'unique_id': 'stkendca17a.fnal.gov-1257314337-25992-0'} Thread tape_thread 00:00:38 stkenmvr193a.fnal.gov 031647 root I LTO4_93MV Done is sent Thread tape_thread 00:00:38 stkenmvr193a.fnal.gov 031647 root I LTO4_93MV maybe_clean Thread tape_thread 00:00:38 stkenmvr193a.fnal.gov 031647 root I LTO4_93MV returned from maybe_clean Thread tape_thread 00:00:38 stkendca17a.fnal.gov 025992 root I ENCP Received final dialog for stkendca17a.fnal.gov-1257314337-25992-0. elapsed=102.610sec 00:00:38 stkendca17a.fnal.gov 025992 root I ENCP leaving transfer_file(). 00:00:39 stkenmvr145a.fnal.gov 022974 root I LTO4_45MV written bytes 3633100288/3633100288, blocks 27719 header 192 trailer 304 Thread tape_thread 00:00:39 stkendca17a.fnal.gov 025992 root I ENCP pf_status = ('ok', None) skip_layer_cleanup = False 00:00:39 stkenmvr197a.fnal.gov 025501 root I LTO4_97MV filemarks written. Tape VON178 absolute location in blocks 4682514 Thread tape_thread 00:00:39 stkenmvr197a.fnal.gov 025501 root I LTO4_97MV write_tape timing:pull 0.0238349437714 write 0.920709609985 crc_check 1.35864448547 freespace 0.0284898281097 block_write 2.33167886734 idle 0.0450367927551 Thread tape_thread 00:00:39 stkenmvr197a.fnal.gov 025501 root I LTO4_97MV drive stats after write. Tape VON178 position 791 block 0 block_size 0 bloc_loc 4682514 tot_blocks 762939000 BOT 0 read_err 0 write_err 0 bytes 289488384 block_write_tot 2.33167886734 tape_rate 124154482.873 Thread tape_thread 00:00:39 stkensrv0n.fnal.gov 028796 enstore I FILSRV Updating update file set deleted = 'n', pnfs_path = '/pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/03/channel02/D0OFPRD1_t701979310_s103226_p22', drive = 'stkenmvr193a:/dev/rmt/tps0d0n:9310097076', pnfs_id = '0030000000000000061D1078' where bfid = 'CDMS125731443800000'; 00:00:39 cmsstor133.fnal.gov 002545 root I ENCP Creating zero length output files. 00:00:39 cmsstor133.fnal.gov 002545 root I ENCP Sending /pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0086/860CB254-EFC8-DE11-9A62-0018F3D09706.root write request to LM: unique_id: cmsstor133.fnal.gov-1257314439-2545-0 inputfile: /storage/data1/write-pool-1/data/000700000000000062546FB0 outputfile: /pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0086/.(access)(000700000000000062546FB0) 00:00:39 stkenmvr197a.fnal.gov 025501 root I LTO4_97MV remaining bytes info in DB 198881280000 estimated 198591791616 reported from drive 198611968000 position 792 Thread tape_thread 00:00:39 stkenmvr197a.fnal.gov 025501 root I LTO4_97MV new bitfile request {'uid': 0, 'external_label': 'VON178', 'gid': 0, 'location_cookie': '0000_000000000_0000791', 'complete_crc': 2723434399L, 'sanity_cookie': (65536L, 1678900690L), 'size': 289487922L} Thread tape_thread 00:00:39 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM MSG_TYPE=ADD_TO_LMQ request added to pending queue for storage group : cms 00:00:39 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM write Q'd /storage/data1/write-pool-1/data/000700000000000062546FB0 (cmsstor133.fnal.gov-1257314439-2545-0) -> /pnfs/fs/usr/cms/WAX/11/store/data/1/Calo/RECO/backfill_CosmicsSeq_v2/0086/860CB254-EFC8-DE11-9A62-0018F3D09706.root : library=CD-LTO4G1 family=cms11 requester:root volume_family:cms.cms11.cpio_odc 00:00:39 cmsstor133.fnal.gov 002545 root I ENCP Listening for control socket at: ('131.225.189.104', 55099) 00:00:39 cmsstor133.fnal.gov 002545 root I ENCP Waiting for mover to connect control socket at address: ('131.225.189.104', 55099) elapsed=1.180sec 00:00:39 stkenmvr197a.fnal.gov 025501 root I LTO4_97MV set remaining: VON178 198611968000 0000_000000000_0000792 Thread tape_thread 00:00:39 last message repeated 1 times 00:00:39 stkensrv0n.fnal.gov 028762 enstore I VOLSRV Updating update volume set last_access = '2009-11-04 00:00:39', eod_cookie = '0000_000000000_0000792', non_del_files = '812', remaining_bytes = 198611968000 where label = 'VON178'; 00:00:39 stkenmvr197a.fnal.gov 025501 root I LTO4_97MV transfer complete volume=VON178 location=792 Thread tape_thread 00:00:39 last message repeated 1 times 00:00:39 stkensrv0n.fnal.gov 028762 enstore I VOLSRV Updating update volume set sum_wr_access = '792', modification_time = '2009-11-04 00:00:39' where label = 'VON178'; 00:00:39 stkenmvr197a.fnal.gov 025501 root I LTO4_97MV Sending done to client: {'vc': {'comment': ' ', 'storage_group': 'minos', 'blocksize': 131072, 'declared': 1225477350.0, 'library': 'CD-LTO4G1', 'si_time': [1242756109.0, 0.0], 'user_inhibit': ['none', 'none'], 'system_inhibit': ['none', 'none'], 'external_label': 'VON178', 'wrapper': 'cpio_odc', 'remaining_bytes': 198881280000L, 'sum_mounts': 496, 'sum_rd_access': 1524, 'file_family_width': 1, 'last_access': 1257314430.0, 'media_type': 'LTO4', 'status': ('ok', None), 'eod_cookie': '0000_000000000_0000791', 'capacity_bytes': 858993459200L, 'non_del_files': 811, 'sum_wr_err': 1, 'sum_wr_access': 791, 'file_family': 'reco_near_dogwood0_sntp', 'address': ('131.225.13.59', 7502), 'volume_family': 'minos.reco_near_dogwood0_sntp.cpio_odc', 'modification_time': 1257314430.0, 'write_protected': 'n', 'sum_rd_err': 1, 'first_access': 1240646517.0}, 'outfilepath': '/pnfs/fnal.gov/usr/minos/reco_near/dogwood0/sntp_data/2009-10/N00017122_0000.cosmic.sntp.dogwood0.0.root', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 1, 'delayed_dismount': None, 'curpri': 1}, 'file_size': 289487922L, 'mover': {'max_buffer': 2621440000L, 'check_written_file': 600, 'max_failures': 5, 'library': 'CD-LTO4G1.library_manager', 'local_mover': 0, 'single_filemark': 1, 'max_consecutive_failures': 3, 'norestart': 'INQ', 'port': 7858, 'mc_device': '0,7,1,13', 'compression': 0, 'serial_num': '9310062655', 'update_interval_in_bound': 2, 'update_interval': 5, 'syslog_entry': 'sense|st[0-9]', 'status': ('ok', None), 'max_rate': 73400320.0, 'vendor_id': 'IBM', 'driver': 'FTTDriver', 'host': 'stkenmvr197a', 'max_time_in_state': 1200, 'device': 'stkenmvr197a:/dev/rmt/tps0d0n', 'name': 'LTO4_97.mover', 'send_stats': 1, 'product_id': 'ULTRIUM-TD4', 'statistics_path': '/tmp/enstore/enstore/LTO4_97.stat', 'mover_address': ('131.225.13.192', 7858), 'mount_delay': 15, 'check_first_written_file': 0, 'callback_addr': ('131.225.13.192', 37253), 'data_ip': 'stkenmvr197a', 'logname': 'LTO4_97MV', 'hostip': '131.225.13.192', 'do_cleaning': 'No', 'media_changer': 'SL8500G1.media_changer'}, 'ignore_fair_share': None, 'retry': None, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fnal.gov/usr/minos/reco_near/dogwood0/sntp_data/2009-10/N00017122_0000.cosmic.sntp.dogwood0.0.root', 'uid': 0, 'uname': 'root', 'mtime': 1257314235, 'rmajor': 0, 'machine': ('Linux', 'stkendca22a.fnal.gov', '2.6.9-89.0.9.ELsmp', '#1 SMP Mon Aug 24 13:47:43 CDT 2009', 'i686'), 'sanity_size': 65536, 'type': 'cpio_odc', 'pstat': (16893, 252364496L, 20L, 1, 42411, 5111, 512L, 1257299891, 1257299891, 1254755314), 'mode_octal': '0100036', 'mode': 33188, 'size_bytes': 289487922L, 'gname': 'root', 'gid': 0, 'fullname': '/diskb/write-pool-2/data/000F0000000000000B20C088', 'inode': 253804680L, 'minor': 20}, 'version': 'v3_8b CVS $Revision: 1.922 $ ', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/diskb/write-pool-2/data/000F0000000000000B20C088', 'status': ('ok', None), 'outfile': '/pnfs/fnal.gov/usr/minos/reco_near/dogwood0/sntp_data/2009-10/.(access)(000F0000000000000B20C088)', 'fc': {'uid': 0, 'external_label': 'VON178', 'gid': 0, 'location_cookie': '0000_000000000_0000791', 'complete_crc': 2723434399L, 'bfid': 'CDMS125731443900000', 'sanity_cookie': (65536L, 1678900690L), 'size': 289487922L}, 'infilepath': '/diskb/write-pool-2/data/000F0000000000000B20C088', 'at_the_top': 292, 'local_inode': 1073746733L, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.13.100', 51703), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'encp_start_time': 1257314234.6061749, 'seek_time': 0.029889106750488281, 'transfer_time': 8.1577229499816895, 't0': 1257314234, 'mount_time': 0.016813993453979492, 'lm_dequeued': 1257314431.1667449, 'job_queued': 1257314235.4269929, 'drive_call_time': 0.88872289657592773, 'in_queue': 195.73944115638733, 'drive_transfer_time': 7.0931122303009033}, 'user_level': 'user2', 'unique_id': 'stkendca22a.fnal.gov-1257314235-10240-0'} Thread tape_thread 00:00:39 stkenmvr197a.fnal.gov 025501 root I LTO4_97MV Done is sent Thread tape_thread 00:00:39 stkenmvr197a.fnal.gov 025501 root I LTO4_97MV maybe_clean Thread tape_thread 00:00:39 stkendca17a.fnal.gov 025992 root I ENCP MSG_TYPE=ENCP_XFER write_to_hsm /diskc/write-pool-3/data/0030000000000000061D1078 -> /pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/03/channel02/D0OFPRD1_t701979310_s103226_p22: 1966080000 bytes copied to VON732 at 58.6 MB/S (74.3 MB/S network) (96.9 MB/S drive) (69.6 MB/S disk) mover=LTO4_93.mover drive_id=ULTRIUM-TD4 drive_sn=9310097076 drive_vendor=IBM elapsed=103.07 {'storage_group': 'exp-db', 'encp_crc': 3747111452L, 'disk_rate': 69.593428509571424, 'drive_rate': 96.850552857579117, 'driver': 'FTTDriver', 'transfer_rate': 58.575487074199366, 'mover_interface': 'stkenmvr193a', 'overall_rate': 57.617858582119858, 'media_changer': 'SL8500G1.media_changer', 'network_rate': 58.575487074199366, 'encp_ip': '131.225.13.5', 'unique_id': 'stkendca17a.fnal.gov-1257314337-25992-0'} 00:00:39 stkendca17a.fnal.gov 025992 root I ENCP File /pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/03/channel02/D0OFPRD1_t701979310_s103226_p22 copied successfully. 00:00:39 stkendca17a.fnal.gov 025992 root I ENCP Completed transferring 1966080000 bytes in 1 files in 103.066860914 sec. Overall rate = 57.6 MB/sec. Transfer rate = 58.6 MB/sec. Network rate = 74.3 MB/sec. Drive rate = 96.9 MB/sec. Disk rate = 69.6 MB/sec. Exit status = 0. 00:00:39 stkendca22a.fnal.gov 010240 root I ENCP Received final dialog for stkendca22a.fnal.gov-1257314235-10240-0. elapsed=204.840sec 00:00:39 stkendca22a.fnal.gov 010240 root I ENCP leaving transfer_file(). 00:00:39 stkenmvr197a.fnal.gov 025501 root I LTO4_97MV returned from maybe_clean Thread tape_thread 00:00:39 stkendca22a.fnal.gov 010240 root I ENCP pf_status = ('ok', None) skip_layer_cleanup = False 00:00:39 stkensrv0n.fnal.gov 028796 enstore I FILSRV Updating update file set deleted = 'n', pnfs_path = '/pnfs/fnal.gov/usr/minos/reco_near/dogwood0/sntp_data/2009-10/N00017122_0000.cosmic.sntp.dogwood0.0.root', drive = 'stkenmvr197a:/dev/rmt/tps0d0n:9310062655', pnfs_id = '000F0000000000000B20C088' where bfid = 'CDMS125731443900000'; 00:00:39 stkendca22a.fnal.gov 010240 root I ENCP MSG_TYPE=ENCP_XFER write_to_hsm /diskb/write-pool-2/data/000F0000000000000B20C088 -> /pnfs/fnal.gov/usr/minos/reco_near/dogwood0/sntp_data/2009-10/N00017122_0000.cosmic.sntp.dogwood0.0.root: 289487922 bytes copied to VON178 at 33.8 MB/S (89.2 MB/S network) (38.9 MB/S drive) (92.4 MB/S disk) mover=LTO4_97.mover drive_id=ULTRIUM-TD4 drive_sn=9310062655 drive_vendor=IBM elapsed=205.09 {'storage_group': 'minos', 'encp_crc': 2723434399L, 'disk_rate': 92.382266000041426, 'drive_rate': 38.921870563650181, 'driver': 'FTTDriver', 'transfer_rate': 33.788034440022251, 'mover_interface': 'stkenmvr197a', 'overall_rate': 32.498997288325455, 'media_changer': 'SL8500G1.media_changer', 'network_rate': 33.788034440022251, 'encp_ip': '131.225.13.100', 'unique_id': 'stkendca22a.fnal.gov-1257314235-10240-0'} 00:00:39 stkendca22a.fnal.gov 010240 root I ENCP File /pnfs/fnal.gov/usr/minos/reco_near/dogwood0/sntp_data/2009-10/N00017122_0000.cosmic.sntp.dogwood0.0.root copied successfully. 00:00:39 stkendca22a.fnal.gov 010240 root I ENCP Completed transferring 289487922 bytes in 1 files in 205.089535952 sec. Overall rate = 32.5 MB/sec. Transfer rate = 33.8 MB/sec. Network rate = 89.2 MB/sec. Drive rate = 38.9 MB/sec. Disk rate = 92.4 MB/sec. Exit status = 0. 00:00:39 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:39 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM write_to_hsm next work on vol=VON732 mover=LTO4_93.mover requester:root 00:00:39 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM HAVE_BOUND:sending write_to_hsm /pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/03/channel01/D0OFPRD1_t701984821_s103234_p14 to mover LTO4_93.mover ('131.225.13.188', 7854) DEL_DISM None 00:00:39 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM MSG_TYPE=ADD_TO_LMQ request added to work at movers queue for storage group : exp-db 00:00:39 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM HAVE_BOUND:sending {'vc': {'storage_group': 'exp-db', 'external_label': 'VON732', 'library': 'CD-LTO4G1', 'file_family': 'daily-d0-offline', 'wrapper': 'cpio_odc', 'address': ('131.225.13.59', 7502), 'volume_family': 'exp-db.daily-d0-offline.cpio_odc', 'file_family_width': 2}, 'outfilepath': '/pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/03/channel01/D0OFPRD1_t701984821_s103234_p14', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 1, 'delayed_dismount': None, 'curpri': 1}, 'file_size': 1966080000L, 'mover': 'LTO4_93.mover', 'ignore_fair_share': None, 'retry': None, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/03/channel01/D0OFPRD1_t701984821_s103234_p14', 'uid': 0, 'uname': 'root', 'type': 'cpio_odc', 'gid': 0, 'mtime': 1257314408, 'rmajor': 0, 'machine': ('Linux', 'stkendca18a.fnal.gov', '2.6.9-89.0.9.ELsmp', '#1 SMP Mon Aug 24 13:47:43 CDT 2009', 'i686'), 'sanity_size': 65536, 'pstat': (16877, 820618152L, 20L, 1, 1602, 2752, 512L, 1257314387, 1257314387, 1254398418), 'mode_octal': '0100016', 'mode': 33188, 'gname': 'root', 'size_bytes': 1966080000L, 'fullname': '/diska/write-pool-1/data/0030000000000000061D10A0', 'inode': 807211168L, 'minor': 20}, 'version': 'v3_8b CVS $Revision: 1.922 $ ', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/diska/write-pool-1/data/0030000000000000061D10A0', 'status': ('ok', None), 'outfile': '/pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/03/channel01/.(access)(0030000000000000061D10A0)', 'fc': {'size': 1966080000L, 'external_label': 'VON732', 'address': ('131.225.13.59', 7501)}, 'infilepath': '/diska/write-pool-1/data/0030000000000000061D10A0', 'at_the_top': 91, 'local_inode': 536897842L, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.13.6', 49482), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'lm_dequeued': 1257314439.832962, 'encp_start_time': 1257314407.357821, 'job_queued': 1257314408.526901, 't0': 1257314407, 'in_queue': 31.305984973907471}, 'user_level': 'user2', 'unique_id': 'stkendca18a.fnal.gov-1257314408-29328-0'} to mover 00:00:39 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:39 stkenmvr193a.fnal.gov 031647 root I LTO4_93MV WRITE_TO_HSM Thread MainThread 00:00:39 stkenmvr193a.fnal.gov 031647 root I LTO4_93MV SENDING {'vc': {'storage_group': 'exp-db', 'external_label': 'VON732', 'library': 'CD-LTO4G1', 'file_family': 'daily-d0-offline', 'wrapper': 'cpio_odc', 'address': ('131.225.13.59', 7502), 'volume_family': 'exp-db.daily-d0-offline.cpio_odc', 'file_family_width': 2}, 'outfilepath': '/pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/03/channel01/D0OFPRD1_t701984821_s103234_p14', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 1, 'delayed_dismount': None, 'curpri': 1}, 'file_size': 1966080000L, 'mover': {'max_buffer': 2621440000L, 'check_written_file': 600, 'max_failures': 5, 'library': 'CD-LTO4G1.library_manager', 'local_mover': 0, 'single_filemark': 1, 'max_consecutive_failures': 3, 'norestart': 'INQ', 'port': 7854, 'mc_device': '0,7,1,11', 'compression': 0, 'serial_num': '9310097076', 'update_interval_in_bound': 2, 'update_interval': 5, 'syslog_entry': 'sense|st[0-9]', 'status': ('ok', None), 'max_rate': 73400320.0, 'vendor_id': 'IBM', 'driver': 'FTTDriver', 'host': 'stkenmvr193a', 'max_time_in_state': 1200, 'device': 'stkenmvr193a:/dev/rmt/tps0d0n', 'name': 'LTO4_93.mover', 'send_stats': 1, 'product_id': 'ULTRIUM-TD4', 'statistics_path': '/tmp/enstore/enstore/LTO4_93.stat', 'mover_address': ('131.225.13.188', 7854), 'mount_delay': 15, 'check_first_written_file': 0, 'callback_addr': ('131.225.13.188', 36326), 'data_ip': 'stkenmvr193a', 'logname': 'LTO4_93MV', 'hostip': '131.225.13.188', 'do_cleaning': 'No', 'media_changer': 'SL8500G1.media_changer'}, 'ignore_fair_share': None, 'retry': None, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/03/channel01/D0OFPRD1_t701984821_s103234_p14', 'uid': 0, 'uname': 'root', 'mtime': 1257314408, 'rmajor': 0, 'machine': ('Linux', 'stkendca18a.fnal.gov', '2.6.9-89.0.9.ELsmp', '#1 SMP Mon Aug 24 13:47:43 CDT 2009', 'i686'), 'sanity_size': 65536, 'type': 'cpio_odc', 'pstat': (16877, 820618152L, 20L, 1, 1602, 2752, 512L, 1257314387, 1257314387, 1254398418), 'mode_octal': '0100016', 'mode': 33188, 'size_bytes': 1966080000L, 'gname': 'root', 'gid': 0, 'fullname': '/diska/write-pool-1/data/0030000000000000061D10A0', 'inode': 807211168L, 'minor': 20}, 'version': 'v3_8b CVS $Revision: 1.922 $ ', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/diska/write-pool-1/data/0030000000000000061D10A0', 'status': ('ok', None), 'outfile': '/pnfs/fnal.gov/usr/exp-db/daily/d0-offline/d0ofprd1/2009/11-November/03/channel01/.(access)(0030000000000000061D10A0)', 'fc': {'size': 1966080000L, 'external_label': 'VON732', 'address': ('131.225.13.59', 7501)}, 'infilepath': '/diska/write-pool-1/data/0030000000000000061D10A0', 'at_the_top': 91, 'local_inode': 536897842L, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.13.6', 49482), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'lm_dequeued': 1257314439.832962, 'encp_start_time': 1257314407.357821, 'job_queued': 1257314408.526901, 't0': 1257314407, 'in_queue': 31.305984973907471}, 'user_level': 'user2', 'unique_id': 'stkendca18a.fnal.gov-1257314408-29328-0'} Thread client_connect_thread 00:00:39 stkendca18a.fnal.gov 029328 root I ENCP Control socket ('131.225.13.6', 49482) is connected to ('131.225.13.188', 36327) for stkendca18a.fnal.gov-1257314408-29328-0. elapsed=32.520sec 00:00:39 stkendca18a.fnal.gov 029328 root I ENCP Received callback ticket from mover LTO4_93.mover for transfer stkendca18a.fnal.gov-1257314408-29328-0. elapsed=32.520sec 00:00:39 stkendca18a.fnal.gov 029328 root I ENCP Atempting to connect data socket to mover at ('131.225.13.188', 36326). 00:00:39 stkendca18a.fnal.gov 029328 root I ENCP Connecting data socket to mover (('131.225.13.188', 36326)) with interface 131.225.13.6. elapsed=32.520sec 00:00:39 stkendca18a.fnal.gov 029328 root I ENCP Data socket ('131.225.13.6', 49484) is connected to ('131.225.13.188', 36326). elapsed=32.520sec 00:00:39 stkendca18a.fnal.gov 029328 root I ENCP Opening local file. 00:00:39 stkenmvr193a.fnal.gov 031647 root I LTO4_93MV tape position: target 236 current 236 0 Thread seek_thread 00:00:39 stkenmvr193a.fnal.gov 031647 root I LTO4_93MV write_tape starting, bytes_to_write=1966080512 Thread tape_thread 00:00:39 stkendca18a.fnal.gov 029328 root I ENCP pf_status = ('ok', None) skip_layer_cleanup = False 00:00:39 stkenmvr193a.fnal.gov 031647 root I LTO4_93MV Write starting. Tape VON732 absolute location in blocks 3400015 Thread tape_thread 00:00:40 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM 00:00:40 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:40 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM write_to_hsm next work on vol=VON178 mover=LTO4_97.mover requester:root 00:00:40 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM HAVE_BOUND:sending write_to_hsm /pnfs/fnal.gov/usr/minos/reco_near/dogwood0/sntp_data/2009-11/N00017122_0021.spill.sntp.dogwood0.0.root to mover LTO4_97.mover ('131.225.13.192', 7858) DEL_DISM None 00:00:40 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM MSG_TYPE=ADD_TO_LMQ request added to work at movers queue for storage group : minos 00:00:40 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM HAVE_BOUND:sending {'vc': {'storage_group': 'minos', 'external_label': 'VON178', 'library': 'CD-LTO4G1', 'file_family': 'reco_near_dogwood0_sntp', 'wrapper': 'cpio_odc', 'address': ('131.225.13.59', 7502), 'volume_family': 'minos.reco_near_dogwood0_sntp.cpio_odc', 'file_family_width': 1}, 'outfilepath': '/pnfs/fnal.gov/usr/minos/reco_near/dogwood0/sntp_data/2009-11/N00017122_0021.spill.sntp.dogwood0.0.root', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 1, 'delayed_dismount': None, 'curpri': 1}, 'file_size': 250430801L, 'mover': 'LTO4_97.mover', 'ignore_fair_share': None, 'retry': None, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fnal.gov/usr/minos/reco_near/dogwood0/sntp_data/2009-11/N00017122_0021.spill.sntp.dogwood0.0.root', 'uid': 0, 'uname': 'root', 'type': 'cpio_odc', 'gid': 0, 'mtime': 1257314235, 'rmajor': 0, 'machine': ('Linux', 'stkendca22a.fnal.gov', '2.6.9-89.0.9.ELsmp', '#1 SMP Mon Aug 24 13:47:43 CDT 2009', 'i686'), 'sanity_size': 65536, 'pstat': (16893, 253805008L, 20L, 1, 42411, 5111, 512L, 1257299960, 1257299960, 1257299926), 'mode_octal': '0100036', 'mode': 33188, 'gname': 'root', 'size_bytes': 250430801L, 'fullname': '/diskb/write-pool-2/data/000F0000000000000B20C300', 'inode': 253805312L, 'minor': 20}, 'version': 'v3_8b CVS $Revision: 1.922 $ ', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/diskb/write-pool-2/data/000F0000000000000B20C300', 'status': ('ok', None), 'outfile': '/pnfs/fnal.gov/usr/minos/reco_near/dogwood0/sntp_data/2009-11/.(access)(000F0000000000000B20C300)', 'fc': {'size': 250430801L, 'external_label': 'VON178', 'address': ('131.225.13.59', 7501)}, 'infilepath': '/diskb/write-pool-2/data/000F0000000000000B20C300', 'at_the_top': 18, 'local_inode': 1073746742L, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.13.100', 51704), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'lm_dequeued': 1257314440.1871481, 'encp_start_time': 1257314234.6241479, 'job_queued': 1257314235.458529, 't0': 1257314234, 'in_queue': 204.72833108901978}, 'user_level': 'user2', 'unique_id': 'stkendca22a.fnal.gov-1257314235-10242-0'} to mover 00:00:40 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:40 stkenmvr197a.fnal.gov 025501 root I LTO4_97MV WRITE_TO_HSM Thread MainThread 00:00:40 stkenmvr197a.fnal.gov 025501 root I LTO4_97MV SENDING {'vc': {'storage_group': 'minos', 'external_label': 'VON178', 'library': 'CD-LTO4G1', 'file_family': 'reco_near_dogwood0_sntp', 'wrapper': 'cpio_odc', 'address': ('131.225.13.59', 7502), 'volume_family': 'minos.reco_near_dogwood0_sntp.cpio_odc', 'file_family_width': 1}, 'outfilepath': '/pnfs/fnal.gov/usr/minos/reco_near/dogwood0/sntp_data/2009-11/N00017122_0021.spill.sntp.dogwood0.0.root', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 1, 'delayed_dismount': None, 'curpri': 1}, 'file_size': 250430801L, 'mover': {'max_buffer': 2621440000L, 'check_written_file': 600, 'max_failures': 5, 'library': 'CD-LTO4G1.library_manager', 'local_mover': 0, 'single_filemark': 1, 'max_consecutive_failures': 3, 'norestart': 'INQ', 'port': 7858, 'mc_device': '0,7,1,13', 'compression': 0, 'serial_num': '9310062655', 'update_interval_in_bound': 2, 'update_interval': 5, 'syslog_entry': 'sense|st[0-9]', 'status': ('ok', None), 'max_rate': 73400320.0, 'vendor_id': 'IBM', 'driver': 'FTTDriver', 'host': 'stkenmvr197a', 'max_time_in_state': 1200, 'device': 'stkenmvr197a:/dev/rmt/tps0d0n', 'name': 'LTO4_97.mover', 'send_stats': 1, 'product_id': 'ULTRIUM-TD4', 'statistics_path': '/tmp/enstore/enstore/LTO4_97.stat', 'mover_address': ('131.225.13.192', 7858), 'mount_delay': 15, 'check_first_written_file': 0, 'callback_addr': ('131.225.13.192', 37255), 'data_ip': 'stkenmvr197a', 'logname': 'LTO4_97MV', 'hostip': '131.225.13.192', 'do_cleaning': 'No', 'media_changer': 'SL8500G1.media_changer'}, 'ignore_fair_share': None, 'retry': None, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fnal.gov/usr/minos/reco_near/dogwood0/sntp_data/2009-11/N00017122_0021.spill.sntp.dogwood0.0.root', 'uid': 0, 'uname': 'root', 'mtime': 1257314235, 'rmajor': 0, 'machine': ('Linux', 'stkendca22a.fnal.gov', '2.6.9-89.0.9.ELsmp', '#1 SMP Mon Aug 24 13:47:43 CDT 2009', 'i686'), 'sanity_size': 65536, 'type': 'cpio_odc', 'pstat': (16893, 253805008L, 20L, 1, 42411, 5111, 512L, 1257299960, 1257299960, 1257299926), 'mode_octal': '0100036', 'mode': 33188, 'size_bytes': 250430801L, 'gname': 'root', 'gid': 0, 'fullname': '/diskb/write-pool-2/data/000F0000000000000B20C300', 'inode': 253805312L, 'minor': 20}, 'version': 'v3_8b CVS $Revision: 1.922 $ ', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/diskb/write-pool-2/data/000F0000000000000B20C300', 'status': ('ok', None), 'outfile': '/pnfs/fnal.gov/usr/minos/reco_near/dogwood0/sntp_data/2009-11/.(access)(000F0000000000000B20C300)', 'fc': {'size': 250430801L, 'external_label': 'VON178', 'address': ('131.225.13.59', 7501)}, 'infilepath': '/diskb/write-pool-2/data/000F0000000000000B20C300', 'at_the_top': 18, 'local_inode': 1073746742L, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.13.100', 51704), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'lm_dequeued': 1257314440.1871481, 'encp_start_time': 1257314234.6241479, 'job_queued': 1257314235.458529, 't0': 1257314234, 'in_queue': 204.72833108901978}, 'user_level': 'user2', 'unique_id': 'stkendca22a.fnal.gov-1257314235-10242-0'} Thread client_connect_thread 00:00:40 stkendca22a.fnal.gov 010242 root I ENCP Control socket ('131.225.13.100', 51704) is connected to ('131.225.13.192', 37256) for stkendca22a.fnal.gov-1257314235-10242-0. elapsed=205.600sec 00:00:40 stkendca22a.fnal.gov 010242 root I ENCP Received callback ticket from mover LTO4_97.mover for transfer stkendca22a.fnal.gov-1257314235-10242-0. elapsed=205.600sec 00:00:40 stkendca22a.fnal.gov 010242 root I ENCP Atempting to connect data socket to mover at ('131.225.13.192', 37255). 00:00:40 stkendca22a.fnal.gov 010242 root I ENCP Connecting data socket to mover (('131.225.13.192', 37255)) with interface 131.225.13.100. elapsed=205.600sec 00:00:40 stkendca22a.fnal.gov 010242 root I ENCP Data socket ('131.225.13.100', 51750) is connected to ('131.225.13.192', 37255). elapsed=205.600sec 00:00:40 stkendca22a.fnal.gov 010242 root I ENCP Opening local file. 00:00:40 stkendca22a.fnal.gov 010242 root I ENCP pf_status = ('ok', None) skip_layer_cleanup = False 00:00:40 stkenmvr197a.fnal.gov 025501 root I LTO4_97MV tape position: target 792 current 792 0 Thread seek_thread 00:00:40 stkenmvr197a.fnal.gov 025501 root I LTO4_97MV write_tape starting, bytes_to_write=250431488 Thread tape_thread 00:00:40 stkenmvr197a.fnal.gov 025501 root I LTO4_97MV Write starting. Tape VON178 absolute location in blocks 4682514 Thread tape_thread 00:00:40 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:40 last message repeated 1 times 00:00:40 stkenmvr140a.fnal.gov 030526 root I LTO4_40MV written bytes 7515166720/7515166720, blocks 57337 header 192 trailer 587 Thread tape_thread 00:00:41 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:41 stkenmvr196a.fnal.gov 027637 root I LTO4_96MV written bytes 2412616192/2412616192, blocks 18407 header 192 trailer 227 Thread tape_thread 00:00:41 cdfenmvr192a.fnal.gov 006151 root I LTO4_92MV filemarks written. Tape VOV138 absolute location in blocks 255552 Thread tape_thread 00:00:41 cdfenmvr192a.fnal.gov 006151 root I LTO4_92MV write_tape timing:pull 0.000139951705933 write 0.00570869445801 crc_check 0.0106887817383 freespace 0.000140428543091 block_write 0.0166778564453 idle 0.000272035598755 Thread tape_thread 00:00:41 cdfenmvr192a.fnal.gov 006151 root I LTO4_92MV drive stats after write. Tape VOV138 position 5191 block 0 block_size 0 bloc_loc 255552 tot_blocks 762939000 BOT 0 read_err 0 write_err 14 bytes 2305024 block_write_tot 0.0166778564453 tape_rate 138208648.549 Thread tape_thread 00:00:41 cdfenmvr192a.fnal.gov 006151 root I LTO4_92MV remaining bytes info in DB 738902016000 estimated 738899710976 reported from drive 738895872000 position 5194 Thread tape_thread 00:00:41 cdfenmvr192a.fnal.gov 006151 root I LTO4_92MV new bitfile request {'uid': 0, 'external_label': 'VOV138', 'gid': 0, 'location_cookie': '0000_000000000_0005192', 'complete_crc': 1479658941L, 'sanity_cookie': (65536L, 2589080699L), 'size': 2304952L} Thread tape_thread 00:00:42 cdfenmvr192a.fnal.gov 006151 root I LTO4_92MV set remaining: VOV138 738895872000 0000_000000000_0005194 Thread tape_thread 00:00:42 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM 00:00:42 last message repeated 1 times 00:00:42 stkensrv0n.fnal.gov 028762 enstore I VOLSRV Updating update volume set last_access = '2009-11-04 00:00:42', eod_cookie = '0000_000000000_0005194', non_del_files = '1732', remaining_bytes = 738895872000 where label = 'VOV138'; 00:00:42 cdfenmvr192a.fnal.gov 006151 root I LTO4_92MV transfer complete volume=VOV138 location=5194 Thread tape_thread 00:00:42 last message repeated 1 times 00:00:42 stkensrv0n.fnal.gov 028762 enstore I VOLSRV Updating update volume set sum_wr_access = '1731', modification_time = '2009-11-04 00:00:42' where label = 'VOV138'; 00:00:42 cdfenmvr192a.fnal.gov 006151 root I LTO4_92MV Sending done to client: {'vc': {'comment': '', 'storage_group': 'ktev', 'blocksize': 131072, 'declared': 1254165875.0, 'library': 'CD-LTO4F1', 'si_time': [0.0, 0.0], 'user_inhibit': ['none', 'none'], 'system_inhibit': ['none', 'none'], 'external_label': 'VOV138', 'wrapper': 'cern', 'remaining_bytes': 738902016000L, 'sum_mounts': 1, 'sum_rd_access': 0, 'file_family_width': 1, 'last_access': 1257314436.0, 'media_type': 'LTO4', 'status': ('ok', None), 'eod_cookie': '0000_000000000_0005191', 'capacity_bytes': 858993459200L, 'non_del_files': 1731, 'sum_wr_err': 0, 'sum_wr_access': 1730, 'file_family': 'ktev_migration-MIGRATION', 'address': ('131.225.13.59', 7502), 'volume_family': 'ktev.ktev_migration-MIGRATION.cern', 'modification_time': 1257314436.0, 'write_protected': 'n', 'sum_rd_err': 0, 'first_access': 1257302588.0}, 'outfilepath': '/pnfs/fs/usr/Migration/ktev/migrated_from_dlt/KQ/KQRC/KQRC44/f1969', 'encp': {'agetime': 30, 'adminpri': 6, 'delpri': 1, 'basepri': 1000007, 'delayed_dismount': 2, 'curpri': 1000007}, 'file_size': 2304952L, 'mover': {'max_buffer': 2621440000L, 'check_written_file': 600, 'max_failures': 5, 'library': 'CD-LTO4F1.library_manager', 'local_mover': 0, 'single_filemark': 1, 'max_consecutive_failures': 3, 'norestart': 'INQ', 'port': 7853, 'mc_device': '1,0,1,1', 'compression': 0, 'serial_num': '1310060800', 'update_interval_in_bound': 2, 'update_interval': 5, 'syslog_entry': 'sense|st[0-9]', 'status': ('ok', None), 'max_rate': 73400320.0, 'vendor_id': 'IBM', 'driver': 'FTTDriver', 'host': 'cdfenmvr192a', 'max_time_in_state': 1200, 'device': 'cdfenmvr192a:/dev/rmt/tps0d0n', 'name': 'LTO4_92.mover', 'send_stats': 1, 'product_id': 'ULTRIUM-TD4', 'statistics_path': '/tmp/enstore/enstore/LTO4_92.stat', 'mover_address': ('131.225.215.76', 7853), 'mount_delay': 15, 'check_first_written_file': 0, 'callback_addr': ('131.225.215.76', 60662), 'data_ip': 'cdfenmvr192a', 'logname': 'LTO4_92MV', 'hostip': '131.225.215.76', 'do_cleaning': 'No', 'media_changer': 'SL8500F1.media_changer'}, 'ignore_fair_share': 1, 'retry': None, 'lm': {'address': ('131.225.13.4', 8542)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fs/usr/ktev/migrated_from_dlt/KQ/KQRC/KQRC44/f1969', 'uid': 0, 'uname': 'root', 'mtime': 1257314424, 'rmajor': 0, 'machine': ('Linux', 'stkendm7a.fnal.gov', '2.6.9-89.0.9.ELsmp', '#1 SMP Mon Aug 24 13:47:43 CDT 2009', 'i686'), 'sanity_size': 65536, 'type': 'cern', 'pstat': (16877, 678574824L, 20L, 1, 0, 0, 512L, 1257314410, 1257314410, 1252942696), 'mode_octal': '0100016', 'mode': 33188, 'size_bytes': 2304952L, 'gname': 'root', 'gid': 0, 'fullname': '/data/data2/Migration_Spool/VO6153:0000_000000000_0032309', 'inode': 675389848L, 'minor': 20}, 'version': 'v3_8 CVS $Revision: 1.963 $ encp.pyc', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/data/data2/Migration_Spool/VO6153:0000_000000000_0032309', 'status': ('ok', None), 'outfile': '/pnfs/fs/usr/Migration/ktev/migrated_from_dlt/KQ/KQRC/KQRC44/.(access)(00280000000000000841A198)', 'fc': {'uid': 0, 'external_label': 'VOV138', 'gid': 0, 'location_cookie': '0000_000000000_0005192', 'complete_crc': 1479658941L, 'bfid': 'CDMS125731444100000', 'sanity_cookie': (65536L, 2589080699L), 'size': 2304952L}, 'infilepath': '/data/data2/Migration_Spool/VO6153:0000_000000000_0032309', 'at_the_top': 39, 'local_inode': 4495350L, 'resend': {'max_retry': 3, 'resubmits': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.13.19', 57451), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'encp_start_time': 1257314424.4886069, 'seek_time': 0.026901960372924805, 'transfer_time': 5.5714521408081055, 't0': 1257314424, 'mount_time': 0.01830291748046875, 'lm_dequeued': 1257314436.4035311, 'job_queued': 1257314424.7389269, 'drive_call_time': 0.016515254974365234, 'in_queue': 11.664335012435913, 'drive_transfer_time': 5.4183065891265869}, 'user_level': 'admin', 'unique_id': 'stkendm7a.fnal.gov-1257314424-18643-24635'} Thread tape_thread 00:00:42 cdfenmvr192a.fnal.gov 006151 root I LTO4_92MV Done is sent Thread tape_thread 00:00:42 cdfenmvr192a.fnal.gov 006151 root I LTO4_92MV maybe_clean Thread tape_thread 00:00:42 stkendm7a.fnal.gov 018643 root I ENCP Received final dialog for stkendm7a.fnal.gov-1257314424-18643-24635. elapsed=154688.510sec Thread WRITE_2 00:00:42 stkendm7a.fnal.gov 018643 root I ENCP leaving transfer_file(). Thread WRITE_2 00:00:42 cdfenmvr192a.fnal.gov 006151 root I LTO4_92MV returned from maybe_clean Thread tape_thread 00:00:42 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:42 stkensrv0n.fnal.gov 028796 enstore I FILSRV Updating update file set deleted = 'n', pnfs_path = '/pnfs/fs/usr/ktev/migrated_from_dlt/KQ/KQRC/KQRC44/f1969', drive = 'cdfenmvr192a:/dev/rmt/tps0d0n:1310060800', pnfs_id = '00280000000000000841A198' where bfid = 'CDMS125731444100000'; 00:00:42 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:42 stkenmvr140a.fnal.gov 030526 root I LTO4_40MV filemarks written. Tape VOR657 absolute location in blocks 4413795 Thread tape_thread 00:00:42 stkenmvr140a.fnal.gov 030526 root I LTO4_40MV write_tape timing:pull 0.770945310593 write 27.8444533348 crc_check 30.0596525669 freespace 0.821115255356 block_write 59.4961664677 idle 1.26954221725 Thread tape_thread 00:00:42 stkenmvr140a.fnal.gov 030526 root I LTO4_40MV drive stats after write. Tape VOR657 position 191 block 0 block_size 0 bloc_loc 4413795 tot_blocks 762939000 BOT 0 read_err 0 write_err 0 bytes 7515166720 block_write_tot 59.4961664677 tape_rate 126313461.29 Thread tape_thread 00:00:42 stkenmvr140a.fnal.gov 030526 root I LTO4_40MV remaining bytes info in DB 248065024000 estimated 240549857280 reported from drive 241046528000 position 192 Thread tape_thread 00:00:42 stkenmvr140a.fnal.gov 030526 root I LTO4_40MV new bitfile request {'uid': 0, 'external_label': 'VOR657', 'gid': 0, 'location_cookie': '0000_000000000_0000191', 'complete_crc': 3768191170L, 'sanity_cookie': (65536L, 4293536134L), 'size': 7515165941L} Thread tape_thread 00:00:42 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM 00:00:42 last message repeated 1 times 00:00:42 stkenmvr140a.fnal.gov 030526 root I LTO4_40MV set remaining: VOR657 241046528000 0000_000000000_0000192 Thread tape_thread 00:00:42 last message repeated 1 times 00:00:42 stkensrv0n.fnal.gov 028762 enstore I VOLSRV Updating update volume set last_access = '2009-11-04 00:00:42', eod_cookie = '0000_000000000_0000192', non_del_files = '191', remaining_bytes = 241046528000 where label = 'VOR657'; 00:00:42 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM write_to_hsm next work on vol=VOV138 mover=LTO4_92.mover requester:root 00:00:42 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM HAVE_BOUND:sending write_to_hsm /pnfs/fs/usr/ktev/migrated_from_dlt/KQ/KQRC/KQRC37/f2692 to mover LTO4_92.mover ('131.225.215.76', 7853) DEL_DISM 2 00:00:42 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM MSG_TYPE=ADD_TO_LMQ request added to work at movers queue for storage group : ktev 00:00:42 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM HAVE_BOUND:sending {'vc': {'storage_group': 'ktev', 'external_label': 'VOV138', 'library': 'CD-LTO4F1', 'file_family': 'ktev_migration-MIGRATION', 'wrapper': 'cern', 'address': ('131.225.13.59', 7502), 'volume_family': 'ktev.ktev_migration-MIGRATION.cern', 'file_family_width': 1}, 'outfilepath': '/pnfs/fs/usr/Migration/ktev/migrated_from_dlt/KQ/KQRC/KQRC37/f2692', 'encp': {'agetime': 30, 'adminpri': 6, 'delpri': 1, 'basepri': 1000007, 'delayed_dismount': 2, 'curpri': 1000007}, 'file_size': 7907500L, 'mover': 'LTO4_