00:00:00 stkenmvr144a.fnal.gov 006329 root I LTO4_44MV tape position: target 39 current 0 1 Thread seek_thread 00:00:00 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:01 stkensrv4n.fnal.gov 003689 enstore I SL8500G1MC PROCESSING listDrives 00:00:01 stkensrv4n.fnal.gov 005843 enstore I SL8500F1MC PROCESSING listDrives 00:00:01 stkensrv4n.fnal.gov 005852 enstore I STKMC PROCESSING listDrives 00:00:01 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:02 stkenmvr207a.fnal.gov 006260 root I LTO4_107MV Ejecting tape Thread media_thread 00:00:02 stkensrv4n.fnal.gov 003689 enstore I SL8500G1MC MC Overflow: 4 mount VOP202 0,5,1,12 00:00:02 stkenmvr151a.fnal.gov 013228 root E LTO4_51MV loadvol ('MEDIA_CHANGER_QUEUE_FULL', 0, 'retry later') Thread media_thread 00:00:02 stkenmvr151a.fnal.gov 013228 root I LTO4_51MV Media Changer returned MEDIA_CHANGER_QUEUE_FULL Thread media_thread 00:00:03 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:03 stkendca19a.fnal.gov 029479 root I ENCP File /diskb/write-pool-2/data/0030000000000000061A0A28 transfered for stkendca19a.fnal.gov-1257140918-29479-0 in 43.6886229515 seconds. 00:00:04 stkenmvr196a.fnal.gov 027637 root I LTO4_96MV written bytes 1966080512/1966080512, blocks 15001 header 190 trailer 322 Thread tape_thread 00:00:04 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:05 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:05 stkenmvr196a.fnal.gov 027637 root I LTO4_96MV filemarks written. Tape VON705 absolute location in blocks 4530543 Thread tape_thread 00:00:05 stkenmvr196a.fnal.gov 027637 root I LTO4_96MV write_tape timing:pull 0.207730770111 write 6.09947443008 crc_check 9.03238081932 freespace 0.146332263947 block_write 15.4859182835 idle 0.267988920212 Thread tape_thread 00:00:05 stkenmvr196a.fnal.gov 027637 root I LTO4_96MV drive stats after write. Tape VON705 position 311 block 0 block_size 0 bloc_loc 4530543 tot_blocks 762939000 BOT 0 read_err 0 write_err 0 bytes 1966080512 block_write_tot 15.4859182835 tape_rate 126959246.201 Thread tape_thread 00:00:05 stkenmvr196a.fnal.gov 027637 root I LTO4_96MV remaining bytes info in DB 224526336000 estimated 222560255488 reported from drive 222691328000 position 312 Thread tape_thread 00:00:05 stkenmvr196a.fnal.gov 027637 root I LTO4_96MV new bitfile request {'uid': 0, 'external_label': 'VON705', 'gid': 0, 'location_cookie': '0000_000000000_0000311', 'complete_crc': 552109237L, 'sanity_cookie': (65536L, 2378827096L), 'size': 1966080000L} Thread tape_thread 00:00:05 stkenmvr196a.fnal.gov 027637 root I LTO4_96MV set remaining: VON705 222691328000 0000_000000000_0000312 Thread tape_thread 00:00:05 last message repeated 1 times 00:00:05 stkensrv0n.fnal.gov 028762 enstore I VOLSRV Updating update volume set last_access = '2009-11-02 00:00:05', eod_cookie = '0000_000000000_0000312', non_del_files = '311', remaining_bytes = 222691328000 where label = 'VON705'; 00:00:06 stkenmvr196a.fnal.gov 027637 root I LTO4_96MV transfer complete volume=VON705 location=312 Thread tape_thread 00:00:06 last message repeated 1 times 00:00:06 stkensrv0n.fnal.gov 028762 enstore I VOLSRV Updating update volume set sum_wr_access = '311', last_access = '2009-11-02 00:00:06', modification_time = '2009-11-02 00:00:06' where label = 'VON705'; 00:00:06 stkenmvr196a.fnal.gov 027637 root I LTO4_96MV Sending done to client: {'vc': {'comment': '', 'storage_group': 'exp-db', 'blocksize': 131072, 'declared': 1247778817.0, 'library': 'CD-LTO4G1', 'si_time': [0.0, 0.0], 'user_inhibit': ['none', 'none'], 'system_inhibit': ['none', 'none'], 'external_label': 'VON705', 'wrapper': 'cpio_odc', 'remaining_bytes': 224526336000L, 'sum_mounts': 12, 'sum_rd_access': 0, 'file_family_width': 2, 'last_access': 1257141558.0, 'media_type': 'LTO4', 'status': ('ok', None), 'eod_cookie': '0000_000000000_0000311', 'capacity_bytes': 858993459200L, 'non_del_files': 310, 'sum_wr_err': 0, 'sum_wr_access': 310, 'file_family': 'monthly-d0-offline', 'address': ('131.225.13.59', 7502), 'volume_family': 'exp-db.monthly-d0-offline.cpio_odc', 'modification_time': 1257141558.0, 'write_protected': 'n', 'sum_rd_err': 0, 'first_access': 1254423740.0}, 'outfilepath': '/pnfs/fnal.gov/usr/exp-db/monthly/d0-offline/d0ofprd1/2009/11-November/01/channel01/D0OFPRD1_t701810604_s103160_p7', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 1, 'delayed_dismount': None, 'curpri': 1}, 'file_size': 1966080000L, 'mover': {'max_buffer': 2621440000L, 'check_written_file': 600, 'max_failures': 5, 'library': 'CD-LTO4G1.library_manager', 'local_mover': 0, 'single_filemark': 1, 'max_consecutive_failures': 3, 'norestart': 'INQ', 'port': 7857, 'mc_device': '0,7,1,2', 'compression': 0, 'serial_num': '9310108548', 'update_interval_in_bound': 2, 'update_interval': 5, 'syslog_entry': 'sense|st[0-9]', 'status': ('ok', None), 'max_rate': 73400320.0, 'vendor_id': 'IBM', 'driver': 'FTTDriver', 'host': 'stkenmvr196a', 'max_time_in_state': 1200, 'device': 'stkenmvr196a:/dev/rmt/tps0d0n', 'name': 'LTO4_96.mover', 'send_stats': 1, 'product_id': 'ULTRIUM-TD4', 'statistics_path': '/tmp/enstore/enstore/LTO4_96.stat', 'mover_address': ('131.225.13.191', 7857), 'mount_delay': 15, 'check_first_written_file': 0, 'callback_addr': ('131.225.13.191', 57736), 'data_ip': 'stkenmvr196a', 'logname': 'LTO4_96MV', 'hostip': '131.225.13.191', 'do_cleaning': 'No', 'media_changer': 'SL8500G1.media_changer'}, 'ignore_fair_share': None, 'retry': None, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fnal.gov/usr/exp-db/monthly/d0-offline/d0ofprd1/2009/11-November/01/channel01/D0OFPRD1_t701810604_s103160_p7', 'uid': 0, 'uname': 'root', 'mtime': 1257140918, 'rmajor': 0, 'machine': ('Linux', 'stkendca19a.fnal.gov', '2.6.9-89.0.9.ELsmp', '#1 SMP Mon Aug 24 13:47:43 CDT 2009', 'i686'), 'sanity_size': 65536, 'type': 'cpio_odc', 'pstat': (16877, 820633536L, 20L, 1, 1602, 2752, 512L, 1257140856, 1257140856, 1254398425), 'mode_octal': '0100016', 'mode': 33188, 'size_bytes': 1966080000L, 'gname': 'root', 'gid': 0, 'fullname': '/diskb/write-pool-2/data/0030000000000000061A0A28', 'inode': 807012904L, 'minor': 20}, 'version': 'v3_8b CVS $Revision: 1.922 $ ', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/diskb/write-pool-2/data/0030000000000000061A0A28', 'status': ('ok', None), 'outfile': '/pnfs/fnal.gov/usr/exp-db/monthly/d0-offline/d0ofprd1/2009/11-November/01/channel01/.(access)(0030000000000000061A0A28)', 'fc': {'uid': 0, 'external_label': 'VON705', 'gid': 0, 'location_cookie': '0000_000000000_0000311', 'complete_crc': 552109237L, 'bfid': 'CDMS125714160500000', 'sanity_cookie': (65536L, 2378827096L), 'size': 1966080000L}, 'infilepath': '/diskb/write-pool-2/data/0030000000000000061A0A28', 'at_the_top': 118, 'local_inode': 549482456L, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.13.11', 39221), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'encp_start_time': 1257140916.6641369, 'seek_time': 0.031038045883178711, 'transfer_time': 45.980721950531006, 't0': 1257140916, 'mount_time': 0.013924837112426758, 'lm_dequeued': 1257141560.028897, 'job_queued': 1257140918.1233389, 'drive_call_time': 5.8983051776885986, 'in_queue': 641.90526509284973, 'drive_transfer_time': 17.500787019729614}, 'user_level': 'user2', 'unique_id': 'stkendca19a.fnal.gov-1257140918-29479-0'} Thread tape_thread 00:00:06 stkenmvr196a.fnal.gov 027637 root I LTO4_96MV Done is sent Thread tape_thread 00:00:06 stkenmvr196a.fnal.gov 027637 root I LTO4_96MV maybe_clean Thread tape_thread 00:00:06 stkenmvr196a.fnal.gov 027637 root I LTO4_96MV returned from maybe_clean Thread tape_thread 00:00:06 stkendca19a.fnal.gov 029479 root I ENCP Received final dialog for stkendca19a.fnal.gov-1257140918-29479-0. elapsed=690.300sec 00:00:06 stkendca19a.fnal.gov 029479 root I ENCP leaving transfer_file(). 00:00:06 stkendca19a.fnal.gov 029479 root I ENCP pf_status = ('ok', None) skip_layer_cleanup = False 00:00:06 stkensrv0n.fnal.gov 023312 enstore I INFSRV start listing all volumes (3) 00:00:06 stkensrv0n.fnal.gov 028796 enstore I FILSRV Updating update file set deleted = 'n', pnfs_path = '/pnfs/fnal.gov/usr/exp-db/monthly/d0-offline/d0ofprd1/2009/11-November/01/channel01/D0OFPRD1_t701810604_s103160_p7', drive = 'stkenmvr196a:/dev/rmt/tps0d0n:9310108548', pnfs_id = '0030000000000000061A0A28' where bfid = 'CDMS125714160500000'; 00:00:07 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:07 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:07 stkendca19a.fnal.gov 029479 root I ENCP MSG_TYPE=ENCP_XFER write_to_hsm /diskb/write-pool-2/data/0030000000000000061A0A28 -> /pnfs/fnal.gov/usr/exp-db/monthly/d0-offline/d0ofprd1/2009/11-November/01/channel01/D0OFPRD1_t701810604_s103160_p7: 1966080000 bytes copied to VON705 at 40.8 MB/S (72.9 MB/S network) (107 MB/S drive) (42.9 MB/S disk) mover=LTO4_96.mover drive_id=ULTRIUM-TD4 drive_sn=9310108548 drive_vendor=IBM elapsed=690.46 {'storage_group': 'exp-db', 'encp_crc': 552109237L, 'disk_rate': 42.934261623211654, 'drive_rate': 107.13803887140664, 'driver': 'FTTDriver', 'transfer_rate': 40.831039627232343, 'mover_interface': 'stkenmvr196a', 'overall_rate': 39.939628203193912, 'media_changer': 'SL8500G1.media_changer', 'network_rate': 40.831039627232343, 'encp_ip': '131.225.13.11', 'unique_id': 'stkendca19a.fnal.gov-1257140918-29479-0'} 00:00:07 stkendca19a.fnal.gov 029479 root I ENCP File /pnfs/fnal.gov/usr/exp-db/monthly/d0-offline/d0ofprd1/2009/11-November/01/channel01/D0OFPRD1_t701810604_s103160_p7 copied successfully. 00:00:07 stkendca19a.fnal.gov 029479 root I ENCP Completed transferring 1966080000 bytes in 1 files in 690.460525036 sec. Overall rate = 39.9 MB/sec. Transfer rate = 40.8 MB/sec. Network rate = 72.9 MB/sec. Drive rate = 107 MB/sec. Disk rate = 42.9 MB/sec. Exit status = 0. 00:00:07 stkensrv4n.fnal.gov 003689 enstore I SL8500G1MC FINISHED listDrives returned ('ok', 0, None, '', '') 00:00:07 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:07 stkensrv4n.fnal.gov 003689 enstore I SL8500G1MC MC Overflow: 4 dismount VOR424 0,6,1,11 00:00:07 stkenmvr200a.fnal.gov 005076 root E LTO4_100MV unloadvol ('MEDIA_CHANGER_QUEUE_FULL', 0, 'retry later') Thread media_thread 00:00:07 stkenmvr200a.fnal.gov 005076 root I LTO4_100MV Media Changer returned MEDIA_CHANGER_QUEUE_FULL Thread media_thread 00:00:08 stkensrv4n.fnal.gov 003689 enstore I SL8500G1MC MC Overflow: 4 mount VOR382 0,6,1,13 00:00:08 stkenmvr204a.fnal.gov 006261 root E LTO4_104MV loadvol ('MEDIA_CHANGER_QUEUE_FULL', 0, 'retry later') Thread media_thread 00:00:08 stkenmvr204a.fnal.gov 006261 root I LTO4_104MV Media Changer returned MEDIA_CHANGER_QUEUE_FULL Thread media_thread 00:00:08 southport.fnal.gov 007300 e898 I ENCP File /Nearline/data01/boone_0022352_0015.fc transfered for southport.fnal.gov-1257141341-7300-1 in 8.97758507729 seconds. 00:00:09 stkenmvr217a.fnal.gov 007075 root I LTO4_117MV written bytes 104866816/104866816, blocks 801 header 160 trailer 276 Thread tape_thread 00:00:09 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:10 stkensrv4n.fnal.gov 003689 enstore I SL8500G1MC MC Overflow: 4 mount VOR381 0,5,1,3 00:00:10 stkenmvr149a.fnal.gov 006318 root E LTO4_49MV loadvol ('MEDIA_CHANGER_QUEUE_FULL', 0, 'retry later') Thread media_thread 00:00:10 stkenmvr149a.fnal.gov 006318 root I LTO4_49MV Media Changer returned MEDIA_CHANGER_QUEUE_FULL Thread media_thread 00:00:11 stkenmvr217a.fnal.gov 007075 root I LTO4_117MV filemarks written. Tape VOO268 absolute location in blocks 3830094 Thread tape_thread 00:00:11 stkenmvr217a.fnal.gov 007075 root I LTO4_117MV write_tape timing:pull 0.00853800773621 write 0.156560897827 crc_check 0.524430513382 freespace 0.00909924507141 block_write 0.698628664017 idle 0.0168287754059 Thread tape_thread 00:00:11 stkenmvr217a.fnal.gov 007075 root I LTO4_117MV drive stats after write. Tape VOO268 position 4918 block 0 block_size 0 bloc_loc 3830094 tot_blocks 762939000 BOT 0 read_err 0 write_err 0 bytes 104866816 block_write_tot 0.698628664017 tape_rate 150103798.2 Thread tape_thread 00:00:11 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:11 stkenmvr217a.fnal.gov 007075 root I LTO4_117MV remaining bytes info in DB 302472192000 estimated 302367325184 reported from drive 302372864000 position 4919 Thread tape_thread 00:00:11 stkenmvr217a.fnal.gov 007075 root I LTO4_117MV new bitfile request {'uid': 3557, 'external_label': 'VOO268', 'original_bfid': 'CDMS125714151100000', 'gid': 5468, 'location_cookie': '0000_000000000_0004918', 'complete_crc': 1862788119L, 'sanity_cookie': (65536L, 1073748687L), 'size': 104866380L} Thread tape_thread 00:00:11 stkensrv0n.fnal.gov 028796 enstore I FILSRV register copy CDMS125714161100000 of original CDMS125714151100000 00:00:11 stkenmvr217a.fnal.gov 007075 root I LTO4_117MV set remaining: VOO268 302372864000 0000_000000000_0004919 Thread tape_thread 00:00:11 last message repeated 1 times 00:00:11 stkensrv0n.fnal.gov 028762 enstore I VOLSRV Updating update volume set last_access = '2009-11-02 00:00:11', eod_cookie = '0000_000000000_0004919', non_del_files = '5447', remaining_bytes = 302372864000 where label = 'VOO268'; 00:00:11 stkenmvr217a.fnal.gov 007075 root I LTO4_117MV transfer complete volume=VOO268 location=4919 Thread tape_thread 00:00:11 last message repeated 1 times 00:00:11 stkensrv0n.fnal.gov 028762 enstore I VOLSRV Updating update volume set sum_wr_access = '4918', modification_time = '2009-11-02 00:00:11' where label = 'VOO268'; 00:00:11 stkenmvr217a.fnal.gov 007075 root I LTO4_117MV Sending done to client: {'vc': {'comment': '', 'storage_group': 'miniboone', 'blocksize': 131072, 'original_file_family': 'TankData', 'library': 'CD-LTO4F1', 'si_time': [0.0, 0.0], 'user_inhibit': ['none', 'none'], 'declared': 1231283649.0, 'system_inhibit': ['none', 'none'], 'external_label': 'VOO268', 'wrapper': 'cpio_odc', 'remaining_bytes': 302472192000L, 'sum_mounts': 4921, 'sum_rd_access': 0, 'file_family_width': 1, 'last_access': 1257140369.0, 'media_type': 'LTO4', 'status': ('ok', None), 'eod_cookie': '0000_000000000_0004918', 'capacity_bytes': 858993459200L, 'non_del_files': 5446, 'sum_wr_err': 0, 'sum_wr_access': 4917, 'file_family': 'TankData_copy_1', 'address': ('131.225.13.59', 7502), 'volume_family': 'miniboone.TankData_copy_1.cpio_odc', 'modification_time': 1257140369.0, 'write_protected': 'n', 'sum_rd_err': 0, 'first_access': 1233091532.0}, 'encp': {'agetime': 30, 'adminpri': 1, 'delpri': 1, 'basepri': 1000003, 'delayed_dismount': 1, 'curpri': 1000003}, 'file_size': 104866380L, 'mover': {'max_buffer': 2621440000L, 'check_written_file': 50, 'max_failures': 5, 'library': 'CD-LTO4F1.library_manager', 'local_mover': 0, 'single_filemark': 1, 'max_consecutive_failures': 3, 'norestart': 'INQ', 'port': 7877, 'mc_device': '1,2,1,8', 'compression': 0, 'serial_num': '1310151225', 'update_interval_in_bound': 2, 'update_interval': 5, 'syslog_entry': 'sense|st[0-9]', 'status': ('ok', None), 'max_rate': 73400320.0, 'vendor_id': 'IBM', 'driver': 'FTTDriver', 'host': 'stkenmvr217a', 'max_time_in_state': 1200, 'device': 'stkenmvr217a:/dev/rmt/tps0d0n', 'name': 'LTO4_117.mover', 'send_stats': 1, 'product_id': 'ULTRIUM-TD4', 'statistics_path': '/tmp/enstore/enstore/LTO4_117.stat', 'mover_address': ('131.225.13.113', 7877), 'mount_delay': 15, 'check_first_written_file': 0, 'callback_addr': ('131.225.13.113', 51523), 'data_ip': 'stkenmvr217a', 'logname': 'LTO4_117MV', 'hostip': '131.225.13.113', 'do_cleaning': 'No', 'media_changer': 'SL8500F1.media_changer'}, 'ignore_fair_share': None, 'retry': None, 'original_unique_id': 'southport.fnal.gov-1257141341-7300-0', 'lm': {'address': ('131.225.13.4', 8542)}, 'wrapper': {'major': 0, 'mode_octal': '0100011', 'pnfsFilename': '/pnfs/miniboone/TankData/run0022000_0022999/run0022350_0022359/boone_0022352_0015.fc', 'uid': 3557, 'sanity_size': 65536, 'gname': 'e898', 'inode': 884203704L, 'mtime': 1257141341, 'machine': ('Linux', 'southport.fnal.gov', '2.4.21-52.ELsmp', '#1 SMP Fri Sep 28 13:31:57 CDT 2007', 'i686'), 'uname': 'e898', 'pstat': (16872, 884201016L, 10L, 1, 3557, 5468, 512L, 1257140258, 1257140258, 1257066290), 'rminor': 0, 'mode': 33188, 'size_bytes': 104866380L, 'rmajor': 0, 'gid': 5468, 'fullname': '/Nearline/data01/boone_0022352_0015.fc', 'type': 'cpio_odc', 'minor': 10}, 'version': 'v3_6c CVS $Revision: 1.822 $ ', 'encp_daq': None, 'client_crc': 1, 'infile': '/Nearline/data01/boone_0022352_0015.fc', 'status': ('ok', None), 'outfile': '/pnfs/miniboone/TankData/run0022000_0022999/run0022350_0022359/boone_0022352_0015.fc', 'fc': {'uid': 3557, 'external_label': 'VOO268', 'original_bfid': 'CDMS125714151100000', 'gid': 5468, 'location_cookie': '0000_000000000_0004918', 'complete_crc': 1862788119L, 'bfid': 'CDMS125714161100000', 'sanity_cookie': (65536L, 1073748687L), 'size': 104866380L}, 'at_the_top': 5, 'copy': 1, 'local_inode': 3498L, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.194.144', 55860), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'encp_start_time': 1257141341.2777719, 'seek_time': 86.741606950759888, 'transfer_time': 11.331287145614624, 't0': 1257141341, 'lm_dequeued': 1257141513.1356871, 'job_queued': 1257141512.037528, 'drive_call_time': 0.15552544593811035, 'in_queue': 1.0981180667877197, 'drive_transfer_time': 2.7741141319274902}, 'unique_id': 'southport.fnal.gov-1257141341-7300-1'} Thread tape_thread 00:00:11 stkenmvr217a.fnal.gov 007075 root I LTO4_117MV Done is sent Thread tape_thread 00:00:11 stkenmvr217a.fnal.gov 007075 root I LTO4_117MV maybe_clean Thread tape_thread 00:00:11 southport.fnal.gov 007300 e898 I ENCP Received final dialog for southport.fnal.gov-1257141341-7300-1. elapsed=270.030sec 00:00:11 southport.fnal.gov 007300 e898 I ENCP Mover CRC: 1862788119 ENCP CRC: 1862788119 00:00:11 stkenmvr217a.fnal.gov 007075 root I LTO4_117MV returned from maybe_clean Thread tape_thread 00:00:11 stkensrv0n.fnal.gov 028796 enstore E FILSRV made_copy(): CDMS125714151100000 does not have copies 00:00:11 stkensrv0n.fnal.gov 028796 enstore I FILSRV Updating update file set deleted = 'n', pnfs_path = '/pnfs/miniboone/TankData/run0022000_0022999/run0022350_0022359/boone_0022352_0015.fc', drive = 'stkenmvr217a:/dev/rmt/tps0d0n:1310151225', pnfs_id = '003400000000000000B3E0B8' where bfid = 'CDMS125714161100000'; 00:00:11 stkensrv4n.fnal.gov 005852 enstore I STKMC FINISHED listDrives returned ('ok', 0, None, '', '') 00:00:11 southport.fnal.gov 007300 e898 I ENCP MSG_TYPE=ENCP_XFER write_to_hsm /Nearline/data01/boone_0022352_0015.fc -> /pnfs/miniboone/TankData/run0022000_0022999/run0022350_0022359/boone_0022352_0015.fc: 104866380 bytes copied to VOO268 at 8.83 MB/S (11.1 MB/S network) (36.1 MB/S drive) (11.1 MB/S disk) mover=LTO4_117.mover drive_id=ULTRIUM-TD4 drive_sn=1310151225 drive_vendor=IBM elapsed=270.15 {'storage_group': 'miniboone', 'encp_crc': 1862788119L, 'disk_rate': 11.140028939418867, 'drive_rate': 36.050561910735425, 'driver': 'FTTDriver', 'transfer_rate': 8.8291983679185009, 'mover_interface': 'stkenmvr217a', 'overall_rate': 1.0179619529774795, 'unique_id': 'southport.fnal.gov-1257141341-7300-1', 'network_rate': 8.8291983679185009, 'encp_ip': '131.225.194.144', 'media_changer': 'SL8500F1.media_changer'} 00:00:11 southport.fnal.gov 007300 e898 I ENCP Completed transferring 209732760 bytes in 2 files in 270.155575991 sec. Overall rate = 0.805 MB/sec. Transfer rate = 8.91 MB/sec. Network rate = 11.2 MB/sec. Drive rate = 37.2 MB/sec. Disk rate = 11.2 MB/sec. Exit status = 0. 00:00:11 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:11 stkenmvr198a.fnal.gov 015662 root I LTO4_98MV Updating stats Thread media_thread 00:00:11 stkenmvr198a.fnal.gov 015662 root I LTO4_98MV writing a tape mark before dismount Thread media_thread 00:00:12 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM 00:00:12 stkensrv4n.fnal.gov 005843 enstore I SL8500F1MC FINISHED listDrives returned ('ok', 0, None, '', '') 00:00:12 cmsstor188.fnal.gov 019677 root I ENCP File /storage/data3/write-pool-3/data/0007000000000000618C8660 transfered for cmsstor188.fnal.gov-1257141582-19677-0 in 29.0169599056 seconds. 00:00:12 stkenmvr146a.fnal.gov 005990 root I LTO4_46MV written bytes 3437403136/3437403136, blocks 26226 header 286 trailer 288 Thread tape_thread 00:00:12 stkensrv4n.fnal.gov 003689 enstore I SL8500G1MC MC Overflow: 4 mount VOP202 0,5,1,12 00:00:12 stkenmvr151a.fnal.gov 013228 root E LTO4_51MV loadvol ('MEDIA_CHANGER_QUEUE_FULL', 0, 'retry later') Thread media_thread 00:00:12 stkenmvr151a.fnal.gov 013228 root I LTO4_51MV Media Changer returned MEDIA_CHANGER_QUEUE_FULL Thread media_thread 00:00:13 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:13 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:13 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:14 stkenmvr146a.fnal.gov 005990 root I LTO4_46MV filemarks written. Tape VOR200 absolute location in blocks 4750529 Thread tape_thread 00:00:14 stkenmvr146a.fnal.gov 005990 root I LTO4_46MV write_tape timing:pull 0.30375957489 write 11.8829996586 crc_check 13.6467134953 freespace 0.331773519516 block_write 26.1652462482 idle 0.52237701416 Thread tape_thread 00:00:14 stkenmvr146a.fnal.gov 005990 root I LTO4_46MV drive stats after write. Tape VOR200 position 198 block 0 block_size 0 bloc_loc 4750529 tot_blocks 762939000 BOT 0 read_err 0 write_err 0 bytes 3437403136 block_write_tot 26.1652462482 tape_rate 131372856.322 Thread tape_thread 00:00:14 stkenmvr146a.fnal.gov 005990 root I LTO4_46MV selective CRC check after writing file Thread tape_thread 00:00:14 stkenmvr146a.fnal.gov 005990 root I LTO4_46MV compression 0 Thread tape_thread 00:00:14 stkenmvr146a.fnal.gov 005990 root I LTO4_46MV tape position: target 198 current 199 0 Thread tape_thread 00:00:14 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM 00:00:14 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:15 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:15 stkendca19a.fnal.gov 029488 root I ENCP File /diskb/write-pool-2/data/0030000000000000061A0738 transfered for stkendca19a.fnal.gov-1257140918-29488-0 in 31.8555471897 seconds. 00:00:15 stkenmvr198a.fnal.gov 015662 root I LTO4_98MV Ejecting tape Thread media_thread 00:00:15 stkensrv0n.fnal.gov 023312 enstore I INFSRV stop listing all volumes (3) 00:00:15 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM 00:00:16 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:16 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM 00:00:17 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:17 stkensrv4n.fnal.gov 003689 enstore I SL8500G1MC MC Overflow: 4 dismount VOR424 0,6,1,11 00:00:17 stkenmvr200a.fnal.gov 005076 root E LTO4_100MV unloadvol ('MEDIA_CHANGER_QUEUE_FULL', 0, 'retry later') Thread media_thread 00:00:17 stkenmvr200a.fnal.gov 005076 root I LTO4_100MV Media Changer returned MEDIA_CHANGER_QUEUE_FULL Thread media_thread 00:00:18 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:18 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM 00:00:18 stkensrv4n.fnal.gov 003689 enstore I SL8500G1MC MC Overflow: 4 mount VOR382 0,6,1,13 00:00:18 stkenmvr204a.fnal.gov 006261 root E LTO4_104MV loadvol ('MEDIA_CHANGER_QUEUE_FULL', 0, 'retry later') Thread media_thread 00:00:18 stkenmvr204a.fnal.gov 006261 root I LTO4_104MV Media Changer returned MEDIA_CHANGER_QUEUE_FULL Thread media_thread 00:00:19 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:20 stkensrv4n.fnal.gov 003689 enstore I SL8500G1MC MC Overflow: 4 mount VOR381 0,5,1,3 00:00:20 stkenmvr149a.fnal.gov 006318 root E LTO4_49MV loadvol ('MEDIA_CHANGER_QUEUE_FULL', 0, 'retry later') Thread media_thread 00:00:20 stkenmvr149a.fnal.gov 006318 root I LTO4_49MV Media Changer returned MEDIA_CHANGER_QUEUE_FULL Thread media_thread 00:00:20 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM 00:00:20 stkenmvr140a.fnal.gov 030526 root I LTO4_40MV written bytes 1966080512/1966080512, blocks 15001 header 191 trailer 321 Thread tape_thread 00:00:21 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:22 stkenmvr140a.fnal.gov 030526 root I LTO4_40MV filemarks written. Tape VON706 absolute location in blocks 4378069 Thread tape_thread 00:00:22 stkenmvr140a.fnal.gov 030526 root I LTO4_40MV write_tape timing:pull 0.20410323143 write 6.92599773407 crc_check 7.80049467087 freespace 0.179307937622 block_write 15.109903574 idle 0.299962043762 Thread tape_thread 00:00:22 stkenmvr140a.fnal.gov 030526 root I LTO4_40MV drive stats after write. Tape VON706 position 299 block 0 block_size 0 bloc_loc 4378069 tot_blocks 762939000 BOT 0 read_err 0 write_err 0 bytes 1966080512 block_write_tot 15.109903574 tape_rate 130118667.03 Thread tape_thread 00:00:22 stkenmvr140a.fnal.gov 030526 root I LTO4_40MV remaining bytes info in DB 244672512000 estimated 242706431488 reported from drive 242846720000 position 300 Thread tape_thread 00:00:22 stkenmvr140a.fnal.gov 030526 root I LTO4_40MV new bitfile request {'uid': 0, 'external_label': 'VON706', 'gid': 0, 'location_cookie': '0000_000000000_0000299', 'complete_crc': 4238616568L, 'sanity_cookie': (65536L, 2480285226L), 'size': 1966080000L} Thread tape_thread 00:00:22 stkenmvr140a.fnal.gov 030526 root I LTO4_40MV set remaining: VON706 242846720000 0000_000000000_0000300 Thread tape_thread 00:00:22 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM 00:00:22 last message repeated 1 times 00:00:22 stkensrv0n.fnal.gov 028762 enstore I VOLSRV Updating update volume set last_access = '2009-11-02 00:00:22', eod_cookie = '0000_000000000_0000300', non_del_files = '299', remaining_bytes = 242846720000 where label = 'VON706'; 00:00:22 stkenmvr140a.fnal.gov 030526 root I LTO4_40MV transfer complete volume=VON706 location=300 Thread tape_thread 00:00:22 last message repeated 1 times 00:00:22 stkensrv0n.fnal.gov 028762 enstore I VOLSRV Updating update volume set sum_wr_access = '299', modification_time = '2009-11-02 00:00:22' where label = 'VON706'; 00:00:22 stkenmvr140a.fnal.gov 030526 root I LTO4_40MV Sending done to client: {'vc': {'comment': '', 'storage_group': 'exp-db', 'blocksize': 131072, 'declared': 1247778817.0, 'library': 'CD-LTO4G1', 'si_time': [0.0, 0.0], 'user_inhibit': ['none', 'none'], 'system_inhibit': ['none', 'none'], 'external_label': 'VON706', 'wrapper': 'cpio_odc', 'remaining_bytes': 244672512000L, 'sum_mounts': 13, 'sum_rd_access': 0, 'file_family_width': 2, 'last_access': 1257141582.0, 'media_type': 'LTO4', 'status': ('ok', None), 'eod_cookie': '0000_000000000_0000299', 'capacity_bytes': 858993459200L, 'non_del_files': 298, 'sum_wr_err': 0, 'sum_wr_access': 298, 'file_family': 'monthly-d0-offline', 'address': ('131.225.13.59', 7502), 'volume_family': 'exp-db.monthly-d0-offline.cpio_odc', 'modification_time': 1257141582.0, 'write_protected': 'n', 'sum_rd_err': 0, 'first_access': 1254423642.0}, 'outfilepath': '/pnfs/fnal.gov/usr/exp-db/monthly/d0-offline/d0ofprd1/2009/11-November/01/channel01/D0OFPRD1_t701808429_s103156_p20', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 1, 'delayed_dismount': None, 'curpri': 1}, 'file_size': 1966080000L, 'mover': {'max_buffer': 2621440000L, 'check_written_file': 600, 'max_failures': 5, 'library': 'CD-LTO4G1.library_manager', 'local_mover': 0, 'dismount_delay': 30, 'max_consecutive_failures': 3, 'norestart': 'INQ', 'port': 7840, 'mc_device': '0,7,1,15', 'compression': 0, 'serial_num': '9310801386', 'update_interval_in_bound': 2, 'update_interval': 5, 'syslog_entry': 'sense|st[0-9]', 'status': ('ok', None), 'max_rate': 73400320.0, 'vendor_id': 'IBM', 'driver': 'FTTDriver', 'host': 'stkenmvr140a', 'max_time_in_state': 1200, 'device': 'stkenmvr140a:/dev/rmt/tps0d0n', 'name': 'LTO4_40.mover', 'send_stats': 1, 'product_id': 'ULTRIUM-TD4', 'statistics_path': '/tmp/enstore/enstore/LTO4_40.stat', 'single_filemark': 1, 'mount_delay': 15, 'mover_address': ('131.225.13.168', 7840), 'check_first_written_file': 0, 'callback_addr': ('131.225.13.168', 41876), 'data_ip': 'stkenmvr140a', 'logname': 'LTO4_40MV', 'hostip': '131.225.13.168', 'do_cleaning': 'No', 'media_changer': 'SL8500G1.media_changer', 'min_buffer': 67108864L}, 'ignore_fair_share': None, 'retry': None, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fnal.gov/usr/exp-db/monthly/d0-offline/d0ofprd1/2009/11-November/01/channel01/D0OFPRD1_t701808429_s103156_p20', 'uid': 0, 'uname': 'root', 'mtime': 1257140918, 'rmajor': 0, 'machine': ('Linux', 'stkendca19a.fnal.gov', '2.6.9-89.0.9.ELsmp', '#1 SMP Mon Aug 24 13:47:43 CDT 2009', 'i686'), 'sanity_size': 65536, 'type': 'cpio_odc', 'pstat': (16877, 820633536L, 20L, 1, 1602, 2752, 512L, 1257140856, 1257140856, 1254398425), 'mode_octal': '0100016', 'mode': 33188, 'size_bytes': 1966080000L, 'gname': 'root', 'gid': 0, 'fullname': '/diskb/write-pool-2/data/0030000000000000061A0738', 'inode': 807012152L, 'minor': 20}, 'version': 'v3_8b CVS $Revision: 1.922 $ ', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/diskb/write-pool-2/data/0030000000000000061A0738', 'status': ('ok', None), 'outfile': '/pnfs/fnal.gov/usr/exp-db/monthly/d0-offline/d0ofprd1/2009/11-November/01/channel01/.(access)(0030000000000000061A0738)', 'fc': {'uid': 0, 'external_label': 'VON706', 'gid': 0, 'location_cookie': '0000_000000000_0000299', 'complete_crc': 4238616568L, 'bfid': 'CDMS125714162200000', 'sanity_cookie': (65536L, 2480285226L), 'size': 1966080000L}, 'infilepath': '/diskb/write-pool-2/data/0030000000000000061A0738', 'at_the_top': 69, 'local_inode': 628418613L, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.13.11', 39222), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'encp_start_time': 1257140917.562227, 'seek_time': 0.034666776657104492, 'transfer_time': 39.154473066329956, 't0': 1257140917, 'mount_time': 0.038582086563110352, 'lm_dequeued': 1257141583.092104, 'job_queued': 1257140918.5651369, 'drive_call_time': 6.7238693237304688, 'in_queue': 664.52690100669861, 'drive_transfer_time': 17.095650672912598}, 'user_level': (trunc. 4144) Thread tape_thread 00:00:22 stkenmvr140a.fnal.gov 030526 root I LTO4_40MV Done is sent Thread tape_thread 00:00:22 stkenmvr140a.fnal.gov 030526 root I LTO4_40MV maybe_clean Thread tape_thread 00:00:22 stkenmvr140a.fnal.gov 030526 root I LTO4_40MV returned from maybe_clean Thread tape_thread 00:00:22 stkendca19a.fnal.gov 029488 root I ENCP Received final dialog for stkendca19a.fnal.gov-1257140918-29488-0. elapsed=704.850sec 00:00:22 stkendca19a.fnal.gov 029488 root I ENCP leaving transfer_file(). 00:00:22 stkendca19a.fnal.gov 029488 root I ENCP pf_status = ('ok', None) skip_layer_cleanup = False 00:00:22 stkensrv0n.fnal.gov 028796 enstore I FILSRV Updating update file set deleted = 'n', pnfs_path = '/pnfs/fnal.gov/usr/exp-db/monthly/d0-offline/d0ofprd1/2009/11-November/01/channel01/D0OFPRD1_t701808429_s103156_p20', drive = 'stkenmvr140a:/dev/rmt/tps0d0n:9310801386', pnfs_id = '0030000000000000061A0738' where bfid = 'CDMS125714162200000'; 00:00:22 stkendca19a.fnal.gov 029488 root I ENCP MSG_TYPE=ENCP_XFER write_to_hsm /diskb/write-pool-2/data/0030000000000000061A0738 -> /pnfs/fnal.gov/usr/exp-db/monthly/d0-offline/d0ofprd1/2009/11-November/01/channel01/D0OFPRD1_t701808429_s103156_p20: 1966080000 bytes copied to VON706 at 47.9 MB/S (76 MB/S network) (110 MB/S drive) (58.9 MB/S disk) mover=LTO4_40.mover drive_id=ULTRIUM-TD4 drive_sn=9310801386 drive_vendor=IBM elapsed=705.17 {'storage_group': 'exp-db', 'encp_crc': 4238616568L, 'disk_rate': 58.869239644868038, 'drive_rate': 109.67701878530225, 'driver': 'FTTDriver', 'transfer_rate': 47.937225465936635, 'mover_interface': 'stkenmvr140a', 'overall_rate': 47.381708326059524, 'media_changer': 'SL8500G1.media_changer', 'network_rate': 47.937225465936635, 'encp_ip': '131.225.13.11', 'unique_id': 'stkendca19a.fnal.gov-1257140918-29488-0'} 00:00:22 stkendca19a.fnal.gov 029488 root I ENCP File /pnfs/fnal.gov/usr/exp-db/monthly/d0-offline/d0ofprd1/2009/11-November/01/channel01/D0OFPRD1_t701808429_s103156_p20 copied successfully. 00:00:22 stkendca19a.fnal.gov 029488 root I ENCP Completed transferring 1966080000 bytes in 1 files in 705.171474934 sec. Overall rate = 47.4 MB/sec. Transfer rate = 47.9 MB/sec. Network rate = 76 MB/sec. Drive rate = 110 MB/sec. Disk rate = 58.9 MB/sec. Exit status = 0. 00:00:22 stkensrv4n.fnal.gov 003689 enstore I SL8500G1MC MC Overflow: 4 mount VOP202 0,5,1,12 00:00:22 stkenmvr151a.fnal.gov 013228 root E LTO4_51MV loadvol ('MEDIA_CHANGER_QUEUE_FULL', 0, 'retry later') Thread media_thread 00:00:22 stkenmvr151a.fnal.gov 013228 root I LTO4_51MV Media Changer returned MEDIA_CHANGER_QUEUE_FULL Thread media_thread 00:00:23 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:23 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:23 stkenmvr145a.fnal.gov 022974 root I LTO4_45MV Tape is ejected Thread media_thread 00:00:23 stkenmvr145a.fnal.gov 022974 root I LTO4_45MV dismounting VOR385 Thread media_thread 00:00:23 stkensrv4n.fnal.gov 003689 enstore I SL8500G1MC MC Overflow: 4 dismount VOR385 0,6,1,3 00:00:23 stkenmvr145a.fnal.gov 022974 root E LTO4_45MV unloadvol ('MEDIA_CHANGER_QUEUE_FULL', 0, 'retry later') Thread media_thread 00:00:23 stkenmvr145a.fnal.gov 022974 root I LTO4_45MV Media Changer returned MEDIA_CHANGER_QUEUE_FULL Thread media_thread 00:00:24 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM 00:00:24 stkendca20a.fnal.gov 032256 root I ENCP File /diskb/write-pool-2/data/003000000000000006199CD0 transfered for stkendca20a.fnal.gov-1257141042-32256-0 in 56.7089340687 seconds. 00:00:24 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM 00:00:25 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:25 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:26 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM 00:00:27 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:27 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:27 stkensrv4n.fnal.gov 003689 enstore I SL8500G1MC MC Overflow: 4 dismount VOR424 0,6,1,11 00:00:27 stkenmvr200a.fnal.gov 005076 root E LTO4_100MV unloadvol ('MEDIA_CHANGER_QUEUE_FULL', 0, 'retry later') Thread media_thread 00:00:27 stkenmvr200a.fnal.gov 005076 root I LTO4_100MV Media Changer returned MEDIA_CHANGER_QUEUE_FULL Thread media_thread 00:00:28 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM 00:00:28 stkensrv4n.fnal.gov 003689 enstore I SL8500G1MC MC Overflow: 4 mount VOR382 0,6,1,13 00:00:28 stkenmvr204a.fnal.gov 006261 root E LTO4_104MV loadvol ('MEDIA_CHANGER_QUEUE_FULL', 0, 'retry later') Thread media_thread 00:00:28 stkenmvr204a.fnal.gov 006261 root I LTO4_104MV Media Changer returned MEDIA_CHANGER_QUEUE_FULL Thread media_thread 00:00:29 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:29 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:29 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:30 stkensrv4n.fnal.gov 003689 enstore I SL8500G1MC MC Overflow: 4 mount VOR381 0,5,1,3 00:00:30 stkenmvr149a.fnal.gov 006318 root E LTO4_49MV loadvol ('MEDIA_CHANGER_QUEUE_FULL', 0, 'retry later') Thread media_thread 00:00:30 stkenmvr149a.fnal.gov 006318 root I LTO4_49MV Media Changer returned MEDIA_CHANGER_QUEUE_FULL Thread media_thread 00:00:30 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM 00:00:30 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:30 stkenmvr202a.fnal.gov 006121 root I LTO4_102MV written bytes 2147475968/2147475968, blocks 16384 header 190 trailer 322 Thread tape_thread 00:00:31 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:31 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:31 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:31 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM 00:00:32 stkenmvr202a.fnal.gov 006121 root I LTO4_102MV filemarks written. Tape VON218 absolute location in blocks 4937999 Thread tape_thread 00:00:32 stkenmvr202a.fnal.gov 006121 root I LTO4_102MV write_tape timing:pull 0.170480251312 write 8.9117743969 crc_check 9.5981669426 freespace 0.155724287033 block_write 18.8361458778 idle 0.280910015106 Thread tape_thread 00:00:32 stkenmvr202a.fnal.gov 006121 root I LTO4_102MV drive stats after write. Tape VON218 position 330 block 0 block_size 0 bloc_loc 4937999 tot_blocks 762939000 BOT 0 read_err 0 write_err 0 bytes 2147475968 block_write_tot 18.8361458778 tape_rate 114008246.8 Thread tape_thread 00:00:32 stkenmvr202a.fnal.gov 006121 root I LTO4_102MV remaining bytes info in DB 175541248000 estimated 173393772032 reported from drive 173529088000 position 331 Thread tape_thread 00:00:32 stkenmvr202a.fnal.gov 006121 root I LTO4_102MV new bitfile request {'uid': 0, 'external_label': 'VON218', 'gid': 0, 'location_cookie': '0000_000000000_0000330', 'complete_crc': 2691218109L, 'sanity_cookie': (65536L, 3789107439L), 'size': 2147475456L} Thread tape_thread 00:00:32 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM 00:00:32 stkenmvr202a.fnal.gov 006121 root I LTO4_102MV set remaining: VON218 173529088000 0000_000000000_0000331 Thread tape_thread 00:00:32 last message repeated 1 times 00:00:32 stkensrv0n.fnal.gov 028762 enstore I VOLSRV Updating update volume set last_access = '2009-11-02 00:00:32', eod_cookie = '0000_000000000_0000331', non_del_files = '376', remaining_bytes = 173529088000 where label = 'VON218'; 00:00:32 stkenmvr202a.fnal.gov 006121 root I LTO4_102MV transfer complete volume=VON218 location=331 Thread tape_thread 00:00:32 last message repeated 1 times 00:00:32 stkensrv0n.fnal.gov 028762 enstore I VOLSRV Updating update volume set sum_wr_access = '330', modification_time = '2009-11-02 00:00:32' where label = 'VON218'; 00:00:32 stkenmvr202a.fnal.gov 006121 root I LTO4_102MV Sending done to client: {'vc': {'comment': '', 'storage_group': 'exp-db', 'blocksize': 131072, 'declared': 1240352603.0, 'library': 'CD-LTO4G1', 'si_time': [0.0, 0.0], 'user_inhibit': ['none', 'none'], 'system_inhibit': ['none', 'none'], 'external_label': 'VON218', 'wrapper': 'cpio_odc', 'remaining_bytes': 175541248000L, 'sum_mounts': 61, 'sum_rd_access': 0, 'file_family_width': 2, 'last_access': 1257141567.0, 'media_type': 'LTO4', 'status': ('ok', None), 'eod_cookie': '0000_000000000_0000330', 'capacity_bytes': 858993459200L, 'non_del_files': 375, 'sum_wr_err': 0, 'sum_wr_access': 329, 'file_family': 'monthly-cdf-offline', 'address': ('131.225.13.59', 7502), 'volume_family': 'exp-db.monthly-cdf-offline.cpio_odc', 'modification_time': 1257141567.0, 'write_protected': 'n', 'sum_rd_err': 0, 'first_access': 1241271535.0}, 'outfilepath': '/pnfs/fnal.gov/usr/exp-db/monthly/cdf-offline/cdfofpr2/2009/11-November/01/channel02/CDFOFPR2_t701800938_s30273_p2', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 1, 'delayed_dismount': None, 'curpri': 1}, 'file_size': 2147475456L, 'mover': {'max_buffer': 2621440000L, 'check_written_file': 600, 'max_failures': 5, 'library': 'CD-LTO4G1.library_manager', 'local_mover': 0, 'single_filemark': 1, 'max_consecutive_failures': 3, 'norestart': 'INQ', 'port': 7862, 'mc_device': '0,6,1,14', 'compression': 0, 'serial_num': '1310051194', 'update_interval_in_bound': 2, 'update_interval': 5, 'syslog_entry': 'sense|st[0-9]', 'status': ('ok', None), 'max_rate': 73400320.0, 'vendor_id': 'IBM', 'driver': 'FTTDriver', 'host': 'stkenmvr202a', 'max_time_in_state': 1200, 'device': 'stkenmvr202a:/dev/rmt/tps0d0n', 'name': 'LTO4_102.mover', 'send_stats': 1, 'product_id': 'ULTRIUM-TD4', 'statistics_path': '/tmp/enstore/enstore/LTO4_102.stat', 'mover_address': ('131.225.13.197', 7862), 'mount_delay': 15, 'check_first_written_file': 0, 'callback_addr': ('131.225.13.197', 57447), 'data_ip': 'stkenmvr202a', 'logname': 'LTO4_102MV', 'hostip': '131.225.13.197', 'do_cleaning': 'No', 'media_changer': 'SL8500G1.media_changer'}, 'ignore_fair_share': None, 'retry': None, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fnal.gov/usr/exp-db/monthly/cdf-offline/cdfofpr2/2009/11-November/01/channel02/CDFOFPR2_t701800938_s30273_p2', 'uid': 0, 'uname': 'root', 'mtime': 1257141042, 'rmajor': 0, 'machine': ('Linux', 'stkendca20a.fnal.gov', '2.6.9-89.0.9.ELsmp', '#1 SMP Mon Aug 24 13:47:43 CDT 2009', 'i686'), 'sanity_size': 65536, 'type': 'cpio_odc', 'pstat': (16877, 820663440L, 20L, 1, 1602, 2752, 512L, 1257129594, 1257129594, 1254398469), 'mode_octal': '0100016', 'mode': 33188, 'size_bytes': 2147475456L, 'gname': 'root', 'gid': 0, 'fullname': '/diskb/write-pool-2/data/003000000000000006199CD0', 'inode': 806984912L, 'minor': 20}, 'version': 'v3_8b CVS $Revision: 1.922 $ ', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/diskb/write-pool-2/data/003000000000000006199CD0', 'status': ('ok', None), 'outfile': '/pnfs/fnal.gov/usr/exp-db/monthly/cdf-offline/cdfofpr2/2009/11-November/01/channel02/.(access)(003000000000000006199CD0)', 'fc': {'uid': 0, 'external_label': 'VON218', 'gid': 0, 'location_cookie': '0000_000000000_0000330', 'complete_crc': 2691218109L, 'bfid': 'CDMS125714163200000', 'sanity_cookie': (65536L, 3789107439L), 'size': 2147475456L}, 'infilepath': '/diskb/write-pool-2/data/003000000000000006199CD0', 'at_the_top': 80, 'local_inode': 271282370L, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.13.12', 57122), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'encp_start_time': 1257141039.8647959, 'seek_time': 0.029808998107910156, 'transfer_time': 65.075636148452759, 't0': 1257141039, 'mount_time': 0.015456914901733398, 'lm_dequeued': 1257141567.419486, 'job_queued': 1257141043.0451601, 'drive_call_time': 8.6977400779724121, 'in_queue': 524.37405300140381, 'drive_transfer_time': 20.89306378364563}, 'user_level': 'user2', 'unique_id': 'stkendca20a.fnal.gov(trunc. 4101) Thread tape_thread 00:00:32 stkenmvr202a.fnal.gov 006121 root I LTO4_102MV Done is sent Thread tape_thread 00:00:32 stkenmvr202a.fnal.gov 006121 root I LTO4_102MV maybe_clean Thread tape_thread 00:00:32 stkenmvr202a.fnal.gov 006121 root I LTO4_102MV returned from maybe_clean Thread tape_thread 00:00:32 stkendca20a.fnal.gov 032256 root I ENCP Received final dialog for stkendca20a.fnal.gov-1257141042-32256-0. elapsed=592.750sec 00:00:32 stkendca20a.fnal.gov 032256 root I ENCP leaving transfer_file(). 00:00:32 stkendca20a.fnal.gov 032256 root I ENCP pf_status = ('ok', None) skip_layer_cleanup = False 00:00:32 stkensrv4n.fnal.gov 003689 enstore I SL8500G1MC MC Overflow: 4 mount VOP202 0,5,1,12 00:00:32 stkenmvr151a.fnal.gov 013228 root E LTO4_51MV loadvol ('MEDIA_CHANGER_QUEUE_FULL', 0, 'retry later') Thread media_thread 00:00:32 stkenmvr151a.fnal.gov 013228 root I LTO4_51MV Media Changer returned MEDIA_CHANGER_QUEUE_FULL Thread media_thread 00:00:32 stkensrv0n.fnal.gov 028796 enstore I FILSRV Updating update file set deleted = 'n', pnfs_path = '/pnfs/fnal.gov/usr/exp-db/monthly/cdf-offline/cdfofpr2/2009/11-November/01/channel02/CDFOFPR2_t701800938_s30273_p2', drive = 'stkenmvr202a:/dev/rmt/tps0d0n:1310051194', pnfs_id = '003000000000000006199CD0' where bfid = 'CDMS125714163200000'; 00:00:33 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:33 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:33 stkendca20a.fnal.gov 032256 root I ENCP MSG_TYPE=ENCP_XFER write_to_hsm /diskb/write-pool-2/data/003000000000000006199CD0 -> /pnfs/fnal.gov/usr/exp-db/monthly/cdf-offline/cdfofpr2/2009/11-November/01/channel02/CDFOFPR2_t701800938_s30273_p2: 2147475456 bytes copied to VON218 at 31.5 MB/S (57.3 MB/S network) (98 MB/S drive) (36.1 MB/S disk) mover=LTO4_102.mover drive_id=ULTRIUM-TD4 drive_sn=1310051194 drive_vendor=IBM elapsed=593.29 {'storage_group': 'exp-db', 'encp_crc': 2691218109L, 'disk_rate': 36.130209529004603, 'drive_rate': 98.022588199969874, 'driver': 'FTTDriver', 'transfer_rate': 31.452804523748568, 'mover_interface': 'stkenmvr202a', 'overall_rate': 31.177139997669201, 'media_changer': 'SL8500G1.media_changer', 'network_rate': 31.452804523748568, 'encp_ip': '131.225.13.12', 'unique_id': 'stkendca20a.fnal.gov-1257141042-32256-0'} 00:00:33 stkendca20a.fnal.gov 032256 root I ENCP File /pnfs/fnal.gov/usr/exp-db/monthly/cdf-offline/cdfofpr2/2009/11-November/01/channel02/CDFOFPR2_t701800938_s30273_p2 copied successfully. 00:00:33 stkendca20a.fnal.gov 032256 root I ENCP Completed transferring 2147475456 bytes in 1 files in 593.291980028 sec. Overall rate = 31.2 MB/sec. Transfer rate = 31.5 MB/sec. Network rate = 57.3 MB/sec. Drive rate = 98 MB/sec. Disk rate = 36.1 MB/sec. Exit status = 0. 00:00:33 stkensrv4n.fnal.gov 003689 enstore I SL8500G1MC MC Overflow: 4 dismount VOR385 0,6,1,3 00:00:33 stkenmvr145a.fnal.gov 022974 root E LTO4_45MV unloadvol ('MEDIA_CHANGER_QUEUE_FULL', 0, 'retry later') Thread media_thread 00:00:33 stkenmvr145a.fnal.gov 022974 root I LTO4_45MV Media Changer returned MEDIA_CHANGER_QUEUE_FULL Thread media_thread 00:00:33 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:33 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM write_to_hsm next work on vol=VON218 mover=LTO4_102.mover requester:root 00:00:33 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM HAVE_BOUND:sending write_to_hsm /pnfs/fnal.gov/usr/exp-db/monthly/cdf-offline/cdfofpr2/2009/11-November/01/channel01/CDFOFPR2_t701801352_s30277_p5 to mover LTO4_102.mover ('131.225.13.197', 7862) DEL_DISM None 00:00:33 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM MSG_TYPE=ADD_TO_LMQ request added to work at movers queue for storage group : exp-db 00:00:33 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM HAVE_BOUND:sending {'vc': {'storage_group': 'exp-db', 'external_label': 'VON218', 'library': 'CD-LTO4G1', 'file_family': 'monthly-cdf-offline', 'wrapper': 'cpio_odc', 'address': ('131.225.13.59', 7502), 'volume_family': 'exp-db.monthly-cdf-offline.cpio_odc', 'file_family_width': 2}, 'outfilepath': '/pnfs/fnal.gov/usr/exp-db/monthly/cdf-offline/cdfofpr2/2009/11-November/01/channel01/CDFOFPR2_t701801352_s30277_p5', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 1, 'delayed_dismount': None, 'curpri': 1}, 'file_size': 2147475456L, 'mover': 'LTO4_102.mover', 'ignore_fair_share': None, 'retry': None, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fnal.gov/usr/exp-db/monthly/cdf-offline/cdfofpr2/2009/11-November/01/channel01/CDFOFPR2_t701801352_s30277_p5', 'uid': 0, 'uname': 'root', 'type': 'cpio_odc', 'gid': 0, 'mtime': 1257141043, 'rmajor': 0, 'machine': ('Linux', 'stkendca20a.fnal.gov', '2.6.9-89.0.9.ELsmp', '#1 SMP Mon Aug 24 13:47:43 CDT 2009', 'i686'), 'sanity_size': 65536, 'pstat': (16877, 820663368L, 20L, 1, 1602, 2752, 512L, 1257130346, 1257130346, 1254398469), 'mode_octal': '0100016', 'mode': 33188, 'gname': 'root', 'size_bytes': 2147475456L, 'fullname': '/diskb/write-pool-2/data/00300000000000000619A558', 'inode': 806987096L, 'minor': 20}, 'version': 'v3_8b CVS $Revision: 1.922 $ ', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/diskb/write-pool-2/data/00300000000000000619A558', 'status': ('ok', None), 'outfile': '/pnfs/fnal.gov/usr/exp-db/monthly/cdf-offline/cdfofpr2/2009/11-November/01/channel01/.(access)(00300000000000000619A558)', 'fc': {'size': 2147475456L, 'external_label': 'VON218', 'address': ('131.225.13.59', 7501)}, 'infilepath': '/diskb/write-pool-2/data/00300000000000000619A558', 'at_the_top': 314, 'local_inode': 271282376L, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.13.12', 57125), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'lm_dequeued': 1257141633.4799759, 'encp_start_time': 1257141041.101377, 'job_queued': 1257141043.5475941, 't0': 1257141041, 'in_queue': 589.93210387229919}, 'user_level': 'user2', 'unique_id': 'stkendca20a.fnal.gov-1257141043-32240-0'} to mover 00:00:33 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:33 stkenmvr202a.fnal.gov 006121 root I LTO4_102MV WRITE_TO_HSM Thread MainThread 00:00:33 stkenmvr202a.fnal.gov 006121 root I LTO4_102MV SENDING {'vc': {'storage_group': 'exp-db', 'external_label': 'VON218', 'library': 'CD-LTO4G1', 'file_family': 'monthly-cdf-offline', 'wrapper': 'cpio_odc', 'address': ('131.225.13.59', 7502), 'volume_family': 'exp-db.monthly-cdf-offline.cpio_odc', 'file_family_width': 2}, 'outfilepath': '/pnfs/fnal.gov/usr/exp-db/monthly/cdf-offline/cdfofpr2/2009/11-November/01/channel01/CDFOFPR2_t701801352_s30277_p5', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 1, 'delayed_dismount': None, 'curpri': 1}, 'file_size': 2147475456L, 'mover': {'max_buffer': 2621440000L, 'check_written_file': 600, 'max_failures': 5, 'library': 'CD-LTO4G1.library_manager', 'local_mover': 0, 'single_filemark': 1, 'max_consecutive_failures': 3, 'norestart': 'INQ', 'port': 7862, 'mc_device': '0,6,1,14', 'compression': 0, 'serial_num': '1310051194', 'update_interval_in_bound': 2, 'update_interval': 5, 'syslog_entry': 'sense|st[0-9]', 'status': ('ok', None), 'max_rate': 73400320.0, 'vendor_id': 'IBM', 'driver': 'FTTDriver', 'host': 'stkenmvr202a', 'max_time_in_state': 1200, 'device': 'stkenmvr202a:/dev/rmt/tps0d0n', 'name': 'LTO4_102.mover', 'send_stats': 1, 'product_id': 'ULTRIUM-TD4', 'statistics_path': '/tmp/enstore/enstore/LTO4_102.stat', 'mover_address': ('131.225.13.197', 7862), 'mount_delay': 15, 'check_first_written_file': 0, 'callback_addr': ('131.225.13.197', 57455), 'data_ip': 'stkenmvr202a', 'logname': 'LTO4_102MV', 'hostip': '131.225.13.197', 'do_cleaning': 'No', 'media_changer': 'SL8500G1.media_changer'}, 'ignore_fair_share': None, 'retry': None, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fnal.gov/usr/exp-db/monthly/cdf-offline/cdfofpr2/2009/11-November/01/channel01/CDFOFPR2_t701801352_s30277_p5', 'uid': 0, 'uname': 'root', 'mtime': 1257141043, 'rmajor': 0, 'machine': ('Linux', 'stkendca20a.fnal.gov', '2.6.9-89.0.9.ELsmp', '#1 SMP Mon Aug 24 13:47:43 CDT 2009', 'i686'), 'sanity_size': 65536, 'type': 'cpio_odc', 'pstat': (16877, 820663368L, 20L, 1, 1602, 2752, 512L, 1257130346, 1257130346, 1254398469), 'mode_octal': '0100016', 'mode': 33188, 'size_bytes': 2147475456L, 'gname': 'root', 'gid': 0, 'fullname': '/diskb/write-pool-2/data/00300000000000000619A558', 'inode': 806987096L, 'minor': 20}, 'version': 'v3_8b CVS $Revision: 1.922 $ ', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/diskb/write-pool-2/data/00300000000000000619A558', 'status': ('ok', None), 'outfile': '/pnfs/fnal.gov/usr/exp-db/monthly/cdf-offline/cdfofpr2/2009/11-November/01/channel01/.(access)(00300000000000000619A558)', 'fc': {'size': 2147475456L, 'external_label': 'VON218', 'address': ('131.225.13.59', 7501)}, 'infilepath': '/diskb/write-pool-2/data/00300000000000000619A558', 'at_the_top': 314, 'local_inode': 271282376L, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.13.12', 57125), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'lm_dequeued': 1257141633.4799759, 'encp_start_time': 1257141041.101377, 'job_queued': 1257141043.5475941, 't0': 1257141041, 'in_queue': 589.93210387229919}, 'user_level': 'user2', 'unique_id': 'stkendca20a.fnal.gov-1257141043-32240-0'} Thread client_connect_thread 00:00:33 stkendca20a.fnal.gov 032240 root I ENCP Control socket ('131.225.13.12', 57125) is connected to ('131.225.13.197', 57456) for stkendca20a.fnal.gov-1257141043-32240-0. elapsed=592.410sec 00:00:33 stkendca20a.fnal.gov 032240 root I ENCP Received callback ticket from mover LTO4_102.mover for transfer stkendca20a.fnal.gov-1257141043-32240-0. elapsed=592.410sec 00:00:33 stkendca20a.fnal.gov 032240 root I ENCP Atempting to connect data socket to mover at ('131.225.13.197', 57455). 00:00:33 stkendca20a.fnal.gov 032240 root I ENCP Connecting data socket to mover (('131.225.13.197', 57455)) with interface 131.225.13.12. elapsed=592.410sec 00:00:33 stkendca20a.fnal.gov 032240 root I ENCP Data socket ('131.225.13.12', 57172) is connected to ('131.225.13.197', 57455). elapsed=592.420sec 00:00:33 stkendca20a.fnal.gov 032240 root I ENCP Opening local file. 00:00:33 stkenmvr202a.fnal.gov 006121 root I LTO4_102MV tape position: target 331 current 331 0 Thread seek_thread 00:00:33 stkenmvr202a.fnal.gov 006121 root I LTO4_102MV write_tape starting, bytes_to_write=2147475968 Thread tape_thread 00:00:33 stkenmvr202a.fnal.gov 006121 root I LTO4_102MV Write starting. Tape VON218 absolute location in blocks 4937999 Thread tape_thread 00:00:33 stkendca20a.fnal.gov 032240 root I ENCP pf_status = ('ok', None) skip_layer_cleanup = False 00:00:34 cmsstor208.fnal.gov 006387 root I ENCP Version: v3_7d CVS $Revision: 1.922 $ encp OS: Linux 2.6.31 x86_64 Release: Rocks release 5.1 (V.I) Scientific Linux SL release 5.2 (Boron) User: root(0) Group: root(0) Euser: root(0) Egroup: root(0) Library: CD-LTO4G1 Storage Group: cms File Family: BACKFILL2 FF Wrapper: cpio_odc FF Width: 15 Current working directory: cmsstor208.fnal.gov:/ Command line: /home/enstore/enstore/bin/encp --verbose 4 --threaded --ecrc --resubmit-timeout 1800 --age-time 30 --delpri 1 --priority 25 --pnfs-mount /pnfs/fs --shortcut --override-path /pnfs/fnal.gov/usr/cms/WAX//11/store/backfill/2/data/IansMagicMushroomSoup-T0Test-AnalyzeThisAndGetAFreeBigFatHairyPhD/Mu_X/RAW/v10_31_pre20replaythingy_v15/000/000/016/F0D80863-60C6-DE11-AF00-000423D99394.root --put-cache 0007000000000000618C8650 /storage/data2/write-pool-2/data/0007000000000000618C8650 00:00:34 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM 00:00:34 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:35 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:35 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:35 cmsstor208.fnal.gov 006387 root I ENCP Creating zero length output files. 00:00:35 cmsstor208.fnal.gov 006387 root I ENCP Sending /pnfs/fnal.gov/usr/cms/WAX//11/store/backfill/2/data/IansMagicMushroomSoup-T0Test-AnalyzeThisAndGetAFreeBigFatHairyPhD/Mu_X/RAW/v10_31_pre20replaythingy_v15/000/000/016/F0D80863-60C6-DE11-AF00-000423D99394.root write request to LM: unique_id: cmsstor208.fnal.gov-1257141635-6387-0 inputfile: /storage/data2/write-pool-2/data/0007000000000000618C8650 outputfile: /pnfs/fnal.gov/usr/cms/WAX//11/store/backfill/2/data/IansMagicMushroomSoup-T0Test-AnalyzeThisAndGetAFreeBigFatHairyPhD/Mu_X/RAW/v10_31_pre20replaythingy_v15/000/000/016/.(access)(0007000000000000618C8650) 00:00:35 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM MSG_TYPE=ADD_TO_LMQ request added to pending queue for storage group : cms 00:00:35 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM write Q'd /storage/data2/write-pool-2/data/0007000000000000618C8650 (cmsstor208.fnal.gov-1257141635-6387-0) -> /pnfs/fnal.gov/usr/cms/WAX//11/store/backfill/2/data/IansMagicMushroomSoup-T0Test-AnalyzeThisAndGetAFreeBigFatHairyPhD/Mu_X/RAW/v10_31_pre20replaythingy_v15/000/000/016/F0D80863-60C6-DE11-AF00-000423D99394.root : library=CD-LTO4G1 family=BACKFILL2 requester:root volume_family:cms.BACKFILL2.cpio_odc 00:00:35 cmsstor208.fnal.gov 006387 root I ENCP Listening for control socket at: ('131.225.191.79', 35253) 00:00:35 cmsstor208.fnal.gov 006387 root I ENCP Waiting for mover to connect control socket at address: ('131.225.191.79', 35253) elapsed=2.240sec 00:00:35 stkensrv0n.fnal.gov 028762 enstore I VOLSRV Assigning volume VOR198 from storage group cms.BACKFILL2.cpio_odc to library CD-LTO4G1, volume family cms.BACKFILL2.cpio_odc 00:00:35 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM write_to_hsm work on vol=VOR198 mover=LTO4_109.mover requester:root 00:00:35 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM MSG_TYPE=ADD_TO_LMQ request added to work at movers queue for storage group : cms 00:00:35 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM IDLE:sending {'vc': {'storage_group': 'cms', 'library': 'CD-LTO4G1', 'file_family': 'BACKFILL2', 'wrapper': 'cpio_odc', 'address': ('131.225.13.59', 7502), 'volume_family': 'cms.BACKFILL2.cpio_odc', 'file_family_width': 15}, 'outfilepath': '/pnfs/fnal.gov/usr/cms/WAX//11/store/backfill/2/data/IansMagicMushroomSoup-T0Test-AnalyzeThisAndGetAFreeBigFatHairyPhD/Mu_X/RAW/v10_31_pre20replaythingy_v15/000/000/016/F0D80863-60C6-DE11-AF00-000423D99394.root', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 25, 'delayed_dismount': None, 'curpri': 25}, 'file_size': 3862068455L, 'mover': 'LTO4_109.mover', 'ignore_fair_share': None, 'retry': None, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fnal.gov/usr/cms/WAX//11/store/backfill/2/data/IansMagicMushroomSoup-T0Test-AnalyzeThisAndGetAFreeBigFatHairyPhD/Mu_X/RAW/v10_31_pre20replaythingy_v15/000/000/016/F0D80863-60C6-DE11-AF00-000423D99394.root', 'uid': 0, 'uname': 'root', 'type': 'cpio_odc', 'gid': 0, 'mtime': 1257141635, 'rmajor': 0, 'machine': ('Linux', 'cmsstor208.fnal.gov', '2.6.31', '#1 SMP Mon Sep 14 09:43:20 CDT 2009', 'x86_64'), 'sanity_size': 65536, 'pstat': (16877, 126646856, 21L, 1, 9811, 5063, 512, 1257141149, 1257141149, 1257140950), 'mode_octal': '0100016', 'mode': 33188, 'gname': 'root', 'size_bytes': 3862068455L, 'fullname': '/storage/data2/write-pool-2/data/0007000000000000618C8650', 'inode': 126649936L, 'minor': 21}, 'version': 'v3_7d CVS $Revision: 1.922 $ encp', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/storage/data2/write-pool-2/data/0007000000000000618C8650', 'status': ('ok', None), 'outfile': '/pnfs/fnal.gov/usr/cms/WAX//11/store/backfill/2/data/IansMagicMushroomSoup-T0Test-AnalyzeThisAndGetAFreeBigFatHairyPhD/Mu_X/RAW/v10_31_pre20replaythingy_v15/000/000/016/.(access)(0007000000000000618C8650)', 'fc': {'size': 3862068455L, 'external_label': 'VOR198', 'address': ('131.225.13.59', 7501)}, 'infilepath': '/storage/data2/write-pool-2/data/0007000000000000618C8650', 'at_the_top': 2, 'local_inode': 3044, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.191.79', 35253), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'lm_dequeued': 1257141635.444773, 'encp_start_time': 1257141633.1226439, 'job_queued': 1257141635.3416181, 't0': 1257141633, 'in_queue': 0.10305500030517578}, 'user_level': 'admin', 'unique_id': 'cmsstor208.fnal.gov-1257141635-6387-0'} to mover 00:00:35 stkenmvr209a.fnal.gov 006118 root I LTO4_109MV WRITE_TO_HSM Thread MainThread 00:00:35 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM can not remove from sg LTO4_109.mover VOR198 00:00:35 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM SG: {'exp-db': [('LTO4_108.mover', 'VON217'), ('LTO4_107.mover', 'VON399'), ('LTO4_102.mover', 'VON218'), ('LTO4_40.mover', 'VON706'), ('LTO4_96.mover', 'VON705')], 'cms': [('LTO4_41.mover', 'VOR202'), ('LTO4_43.mover', 'VOR416'), ('LTO4_100.mover', 'VOR424'), ('LTO4_49.mover', 'VOR381'), ('LTO4_51.mover', 'VOP202'), ('LTO4_105.mover', 'VOR422'), ('LTO4_110.mover', 'VOR407'), ('LTO4_46.mover', 'VOR200'), ('LTO4_104.mover', 'VOR382'), ('LTO4_45.mover', 'VOR385'), ('LTO4_103.mover', 'VOR423'), ('LTO4_44.mover', 'VOR426'), ('LTO4_101.mover', 'VOR413'), ('LTO4_98.mover', 'VOR201')]} 00:00:35 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM can not remove from vf LTO4_109.mover VOR198 00:00:35 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM VF: {'cms.NonCustodial.cpio_odc': [('LTO4_51.mover', 'VOP202')], 'exp-db.archive-d0-offline.cpio_odc': [('LTO4_107.mover', 'VON399')], 'exp-db.monthly-cdf-offline.cpio_odc': [('LTO4_108.mover', 'VON217'), ('LTO4_102.mover', 'VON218')], 'exp-db.monthly-d0-offline.cpio_odc': [('LTO4_40.mover', 'VON706'), ('LTO4_96.mover', 'VON705')], 'cms.BACKFILL2.cpio_odc': [('LTO4_41.mover', 'VOR202'), ('LTO4_43.mover', 'VOR416'), ('LTO4_100.mover', 'VOR424'), ('LTO4_49.mover', 'VOR381'), ('LTO4_105.mover', 'VOR422'), ('LTO4_110.mover', 'VOR407'), ('LTO4_46.mover', 'VOR200'), ('LTO4_104.mover', 'VOR382'), ('LTO4_45.mover', 'VOR385'), ('LTO4_103.mover', 'VOR423'), ('LTO4_44.mover', 'VOR426'), ('LTO4_101.mover', 'VOR413'), ('LTO4_98.mover', 'VOR201')]} 00:00:35 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:35 stkenmvr209a.fnal.gov 006118 root I LTO4_109MV SENDING {'vc': {'storage_group': 'cms', 'library': 'CD-LTO4G1', 'file_family': 'BACKFILL2', 'wrapper': 'cpio_odc', 'address': ('131.225.13.59', 7502), 'volume_family': 'cms.BACKFILL2.cpio_odc', 'file_family_width': 15}, 'outfilepath': '/pnfs/fnal.gov/usr/cms/WAX//11/store/backfill/2/data/IansMagicMushroomSoup-T0Test-AnalyzeThisAndGetAFreeBigFatHairyPhD/Mu_X/RAW/v10_31_pre20replaythingy_v15/000/000/016/F0D80863-60C6-DE11-AF00-000423D99394.root', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 25, 'delayed_dismount': None, 'curpri': 25}, 'file_size': 3862068455L, 'mover': {'max_buffer': 2621440000L, 'check_written_file': 600, 'max_failures': 5, 'library': 'CD-LTO4G1.library_manager', 'local_mover': 0, 'single_filemark': 1, 'max_consecutive_failures': 3, 'norestart': 'INQ', 'port': 7869, 'mc_device': '0,5,1,7', 'compression': 0, 'serial_num': '9310025229', 'update_interval_in_bound': 2, 'update_interval': 5, 'syslog_entry': 'sense|st[0-9]', 'status': ('ok', None), 'max_rate': 73400320.0, 'vendor_id': 'IBM', 'driver': 'FTTDriver', 'host': 'stkenmvr209a', 'max_time_in_state': 1200, 'device': 'stkenmvr209a:/dev/rmt/tps0d0n', 'name': 'LTO4_109.mover', 'send_stats': 1, 'product_id': 'ULTRIUM-TD4', 'statistics_path': '/tmp/enstore/enstore/LTO4_109.stat', 'mover_address': ('131.225.13.205', 7869), 'mount_delay': 15, 'check_first_written_file': 0, 'callback_addr': ('131.225.13.205', 58034), 'data_ip': 'stkenmvr209a', 'logname': 'LTO4_109MV', 'hostip': '131.225.13.205', 'do_cleaning': 'No', 'media_changer': 'SL8500G1.media_changer'}, 'ignore_fair_share': None, 'retry': None, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fnal.gov/usr/cms/WAX//11/store/backfill/2/data/IansMagicMushroomSoup-T0Test-AnalyzeThisAndGetAFreeBigFatHairyPhD/Mu_X/RAW/v10_31_pre20replaythingy_v15/000/000/016/F0D80863-60C6-DE11-AF00-000423D99394.root', 'uid': 0, 'uname': 'root', 'mtime': 1257141635, 'rmajor': 0, 'machine': ('Linux', 'cmsstor208.fnal.gov', '2.6.31', '#1 SMP Mon Sep 14 09:43:20 CDT 2009', 'x86_64'), 'sanity_size': 65536, 'type': 'cpio_odc', 'pstat': (16877, 126646856, 21L, 1, 9811, 5063, 512, 1257141149, 1257141149, 1257140950), 'mode_octal': '0100016', 'mode': 33188, 'size_bytes': 3862068455L, 'gname': 'root', 'gid': 0, 'fullname': '/storage/data2/write-pool-2/data/0007000000000000618C8650', 'inode': 126649936L, 'minor': 21}, 'version': 'v3_7d CVS $Revision: 1.922 $ encp', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/storage/data2/write-pool-2/data/0007000000000000618C8650', 'status': ('ok', None), 'outfile': '/pnfs/fnal.gov/usr/cms/WAX//11/store/backfill/2/data/IansMagicMushroomSoup-T0Test-AnalyzeThisAndGetAFreeBigFatHairyPhD/Mu_X/RAW/v10_31_pre20replaythingy_v15/000/000/016/.(access)(0007000000000000618C8650)', 'fc': {'size': 3862068455L, 'external_label': 'VOR198', 'address': ('131.225.13.59', 7501)}, 'infilepath': '/storage/data2/write-pool-2/data/0007000000000000618C8650', 'at_the_top': 2, 'local_inode': 3044, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.191.79', 35253), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'lm_dequeued': 1257141635.444773, 'encp_start_time': 1257141633.1226439, 'job_queued': 1257141635.3416181, 't0': 1257141633, 'in_queue': 0.10305500030517578}, 'user_level': 'admin', 'unique_id': 'cmsstor208.fnal.gov-1257141635-6387-0'} Thread client_connect_thread 00:00:35 cmsstor208.fnal.gov 006387 root I ENCP Control socket ('131.225.191.79', 35253) is connected to ('131.225.13.205', 58035) for cmsstor208.fnal.gov-1257141635-6387-0. elapsed=2.360sec 00:00:35 cmsstor208.fnal.gov 006387 root I ENCP Received callback ticket from mover LTO4_109.mover for transfer cmsstor208.fnal.gov-1257141635-6387-0. elapsed=2.370sec 00:00:35 cmsstor208.fnal.gov 006387 root I ENCP Atempting to connect data socket to mover at ('131.225.13.205', 58034). 00:00:35 cmsstor208.fnal.gov 006387 root I ENCP Connecting data socket to mover (('131.225.13.205', 58034)) with interface 131.225.191.79. elapsed=2.370sec 00:00:35 cmsstor208.fnal.gov 006387 root I ENCP Data socket ('131.225.191.79', 53419) is connected to ('131.225.13.205', 58034). elapsed=2.370sec 00:00:35 cmsstor208.fnal.gov 006387 root I ENCP Opening local file. 00:00:35 stkenmvr209a.fnal.gov 006118 root I LTO4_109MV MSG_TYPE=MC_LOAD_REQ mounting VOR198 ULTRIUM-TD4 00:00:35 Thread media_thread 00:00:35 stkensrv4n.fnal.gov 003689 enstore I SL8500G1MC MC Overflow: 4 mount VOR198 0,5,1,7 00:00:35 stkenmvr209a.fnal.gov 006118 root E LTO4_109MV loadvol ('MEDIA_CHANGER_QUEUE_FULL', 0, 'retry later') Thread media_thread 00:00:35 stkenmvr209a.fnal.gov 006118 root I LTO4_109MV Media Changer returned MEDIA_CHANGER_QUEUE_FULL Thread media_thread 00:00:35 cmsstor208.fnal.gov 006387 root I ENCP pf_status = ('ok', None) skip_layer_cleanup = False 00:00:35 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:36 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM 00:00:37 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:37 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:37 stkensrv4n.fnal.gov 003689 enstore I SL8500G1MC MC Overflow: 4 dismount VOR424 0,6,1,11 00:00:37 stkenmvr200a.fnal.gov 005076 root E LTO4_100MV unloadvol ('MEDIA_CHANGER_QUEUE_FULL', 0, 'retry later') Thread media_thread 00:00:37 stkenmvr200a.fnal.gov 005076 root I LTO4_100MV Media Changer returned MEDIA_CHANGER_QUEUE_FULL Thread media_thread 00:00:38 stkendca20a.fnal.gov 032266 root I ENCP File /diskb/write-pool-2/data/003000000000000006199E08 transfered for stkendca20a.fnal.gov-1257141043-32266-0 in 63.9271240234 seconds. 00:00:38 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM 00:00:38 stkendca23a.fnal.gov 023854 root I ENCP Version: v3_8b CVS $Revision: 1.922 $ OS: Linux 2.6.9-89.0.9.ELsmp i686 Release: Scientific Linux Fermi LTS release 4.4 (Wilson), User: root(0) Group: root(0) Euser: root(0) Egroup: root(0) Current working directory: stkendca23a.fnal.gov:/opt Command line: /fnal/ups/prd/encp/v3_8b/Linux-2-6-dcache/encp --verbose 4 --threaded --ecrc --bypass-filesystem-max-filesize-check --age-time 60 --delpri 10 --pnfs-mount /pnfs/fs --shortcut --get-cache 000F0000000000000A9848A0 /diskb/read-pool-2/data/000F0000000000000A9848A0 00:00:38 stkensrv4n.fnal.gov 003689 enstore I SL8500G1MC MC Overflow: 4 mount VOR382 0,6,1,13 00:00:38 stkenmvr204a.fnal.gov 006261 root E LTO4_104MV loadvol ('MEDIA_CHANGER_QUEUE_FULL', 0, 'retry later') Thread media_thread 00:00:38 stkenmvr204a.fnal.gov 006261 root I LTO4_104MV Media Changer returned MEDIA_CHANGER_QUEUE_FULL Thread media_thread 00:00:38 stkendca23a.fnal.gov 023854 root I ENCP Creating zero length output files. 00:00:38 stkendca23a.fnal.gov 023854 root I ENCP Sending /pnfs/fs/usr/.(access)(000F0000000000000A9848A0) read request to LM: unique_id: stkendca23a.fnal.gov-1257141638-23854-0 inputfile: /pnfs/fs/usr/.(access)(000F0000000000000A983D10)/.(access)(000F0000000000000A9848A0) outputfile: /diskb/read-pool-2/data/000F0000000000000A9848A0 00:00:38 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM MSG_TYPE=ADD_TO_LMQ request added to pending queue for storage group : minos 00:00:38 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM read Q'd /pnfs/fs/usr/.(access)(000F0000000000000A9848A0) (stkendca23a.fnal.gov-1257141638-23854-0) -> /diskb/read-pool-2/data/000F0000000000000A9848A0 : library=CD-LTO4G1 family=minos.reco_far_dogwood1_bntp.cpio_odc requester:root 00:00:38 stkendca23a.fnal.gov 023854 root I ENCP Listening for control socket at: ('131.225.13.104', 43614) 00:00:38 stkendca23a.fnal.gov 023854 root I ENCP Waiting for mover to connect control socket at address: ('131.225.13.104', 43614) elapsed=0.580sec 00:00:38 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM read_from_hsm work on vol=VON584 mover=LTO4_48.mover requester:root 00:00:38 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM MSG_TYPE=ADD_TO_LMQ request added to work at movers queue for storage group : minos 00:00:38 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM IDLE:sending {'vc': {'comment': '', 'declared': 1245097650.0, 'blocksize': 131072, 'storage_group': 'minos', 'library': 'CD-LTO4G1', 'eod_cookie': '0000_000000000_0000839', 'current_location': '0000_000000000_0000557', 'user_inhibit': ['none', 'none'], 'system_inhibit': ['none', 'none'], 'external_label': 'VON584', 'wrapper': 'cpio_odc', 'remaining_bytes': 694611968000L, 'sum_mounts': 778, 'sum_rd_access': 1841, 'media_type': 'LTO4', 'last_access': 1257141276.0, 'status': ('ok', None), 'si_time': [0.0, 0.0], 'capacity_bytes': 858993459200L, 'non_del_files': 840, 'sum_wr_err': 0, 'sum_wr_access': 838, 'file_family': 'reco_far_dogwood1_bntp', 'address': ('131.225.13.59', 7502), 'volume_family': 'minos.reco_far_dogwood1_bntp.cpio_odc', 'modification_time': 1254424273.0, 'write_protected': 'n', 'sum_rd_err': 0, 'first_access': 1249952774.0}, 'outfilepath': '/diskb/read-pool-2/data/000F0000000000000A9848A0', 'encp': {'agetime': 60, 'adminpri': -1, 'delpri': 10, 'basepri': 1, 'delayed_dismount': None, 'curpri': 1}, 'file_size': 127633279L, 'mover': 'LTO4_48.mover', 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fs/usr/.(access)(000F0000000000000A9848A0)', 'uid': 0, 'uname': 'root', 'gid': 0, 'rmajor': 0, 'machine': ('Linux', 'stkendca23a.fnal.gov', '2.6.9-89.0.9.ELsmp', '#1 SMP Mon Aug 24 13:47:43 CDT 2009', 'i686'), 'sanity_size': 65536, 'pstat': (33188, 261638304L, 20L, 1, 42411, 5111, 127633279L, 1255146250, 1250454867, 1250451128), 'mode_octal': '0100744', 'mode': 33252, 'gname': 'root', 'size_bytes': 127633279L, 'fullname': '/diskb/read-pool-2/data/000F0000000000000A9848A0', 'inode': 261638304L, 'minor': 20}, 'version': 'v3_8b CVS $Revision: 1.922 $ ', 'encp_daq': None, 'client_crc': 1, 'infile': '/pnfs/fs/usr/.(access)(000F0000000000000A983D10)/.(access)(000F0000000000000A9848A0)', 'status': ('ok', None), 'volume': 'VON584', 'outfile': '/diskb/read-pool-2/data/000F0000000000000A9848A0', 'fc': {'status': ('ok', None), 'pnfsid': '000F0000000000000A9848A0', 'pnfs_name0': '/pnfs/fnal.gov/usr/minos/reco_far/dogwood1/.bntp_data/2009-05/F00043546_0000.spill.bntp.dogwood1.0.root', 'deleted': 'no', 'external_label': 'VON584', 'drive': 'stkenmvr146a:/dev/rmt/tps0d0n:9311032672', 'update': '2009-08-16 15:34:27.42089', 'gid': 0, 'location_cookie': '0000_000000000_0000557', 'complete_crc': 3824704097L, 'address': ('131.225.13.59', 7501), 'uid': 0, 'bfid': 'CDMS125045486600000', 'sanity_cookie': (65536L, 288456852L), 'size': 127633279L}, 'infilepath': '/pnfs/fs/usr/.(access)(000F0000000000000A9848A0)', 'at_the_top': 2, 'bfid': 'CDMS125045486600000', 'local_inode': 1073745108L, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.13.104', 43614), 'work': 'read_from_hsm', 'override_ro_mount': 0, 'times': {'lm_dequeued': 1257141638.781302, 'encp_start_time': 1257141638.1616769, 'job_queued': 1257141638.712321, 't0': 1257141638, 'in_queue': 0.068939924240112305}, 'user_level': 'user2', 'unique_id': 'stkendca23a.fnal.gov-1257141638-23854-0'} to mover 00:00:38 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM can not remove from sg LTO4_48.mover VON584 00:00:38 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM SG: {'exp-db': [('LTO4_108.mover', 'VON217'), ('LTO4_107.mover', 'VON399'), ('LTO4_102.mover', 'VON218'), ('LTO4_40.mover', 'VON706'), ('LTO4_96.mover', 'VON705')], 'cms': [('LTO4_41.mover', 'VOR202'), ('LTO4_43.mover', 'VOR416'), ('LTO4_100.mover', 'VOR424'), ('LTO4_49.mover', 'VOR381'), ('LTO4_51.mover', 'VOP202'), ('LTO4_105.mover', 'VOR422'), ('LTO4_110.mover', 'VOR407'), ('LTO4_46.mover', 'VOR200'), ('LTO4_104.mover', 'VOR382'), ('LTO4_45.mover', 'VOR385'), ('LTO4_103.mover', 'VOR423'), ('LTO4_44.mover', 'VOR426'), ('LTO4_101.mover', 'VOR413'), ('LTO4_98.mover', 'VOR201'), ('LTO4_109.mover', 'VOR198')]} 00:00:38 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM can not remove from vf LTO4_48.mover VON584 00:00:38 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM VF: {'cms.NonCustodial.cpio_odc': [('LTO4_51.mover', 'VOP202')], 'exp-db.archive-d0-offline.cpio_odc': [('LTO4_107.mover', 'VON399')], 'exp-db.monthly-cdf-offline.cpio_odc': [('LTO4_108.mover', 'VON217'), ('LTO4_102.mover', 'VON218')], 'exp-db.monthly-d0-offline.cpio_odc': [('LTO4_40.mover', 'VON706'), ('LTO4_96.mover', 'VON705')], 'cms.BACKFILL2.cpio_odc': [('LTO4_41.mover', 'VOR202'), ('LTO4_43.mover', 'VOR416'), ('LTO4_100.mover', 'VOR424'), ('LTO4_49.mover', 'VOR381'), ('LTO4_105.mover', 'VOR422'), ('LTO4_110.mover', 'VOR407'), ('LTO4_46.mover', 'VOR200'), ('LTO4_104.mover', 'VOR382'), ('LTO4_45.mover', 'VOR385'), ('LTO4_103.mover', 'VOR423'), ('LTO4_44.mover', 'VOR426'), ('LTO4_101.mover', 'VOR413'), ('LTO4_98.mover', 'VOR201'), ('LTO4_109.mover', 'VOR198')]} 00:00:38 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:38 stkenmvr148a.fnal.gov 006010 root I LTO4_48MV READ FROM HSM Thread MainThread 00:00:38 stkenmvr148a.fnal.gov 006010 root I LTO4_48MV SENDING {'vc': {'comment': '', 'declared': 1245097650.0, 'blocksize': 131072, 'storage_group': 'minos', 'library': 'CD-LTO4G1', 'eod_cookie': '0000_000000000_0000839', 'current_location': '0000_000000000_0000557', 'user_inhibit': ['none', 'none'], 'sum_rd_access': 1841, 'system_inhibit': ['none', 'none'], 'external_label': 'VON584', 'wrapper': 'cpio_odc', 'remaining_bytes': 694611968000L, 'sum_mounts': 778, 'capacity_bytes': 858993459200L, 'media_type': 'LTO4', 'last_access': 1257141276.0, 'status': ('ok', None), 'si_time': [0.0, 0.0], 'non_del_files': 840, 'sum_wr_err': 0, 'sum_wr_access': 838, 'file_family': 'reco_far_dogwood1_bntp', 'address': ('131.225.13.59', 7502), 'volume_family': 'minos.reco_far_dogwood1_bntp.cpio_odc', 'modification_time': 1254424273.0, 'write_protected': 'n', 'sum_rd_err': 0, 'first_access': 1249952774.0}, 'outfilepath': '/diskb/read-pool-2/data/000F0000000000000A9848A0', 'encp': {'agetime': 60, 'adminpri': -1, 'delpri': 10, 'basepri': 1, 'delayed_dismount': None, 'curpri': 1}, 'file_size': 127633279L, 'mover': {'max_buffer': 2621440000L, 'check_written_file': 600, 'max_failures': 5, 'library': 'CD-LTO4G1.library_manager', 'local_mover': 0, 'dismount_delay': 30, 'max_consecutive_failures': 3, 'norestart': 'INQ', 'port': 7848, 'mc_device': '0,5,1,15', 'compression': 0, 'serial_num': '9310036898', 'update_interval_in_bound': 2, 'update_interval': 5, 'syslog_entry': 'sense|st[0-9]', 'status': ('ok', None), 'max_rate': 73400320.0, 'vendor_id': 'IBM', 'driver': 'FTTDriver', 'host': 'stkenmvr148a', 'max_time_in_state': 1200, 'device': 'stkenmvr148a:/dev/rmt/tps0d0n', 'name': 'LTO4_48.mover', 'send_stats': 1, 'product_id': 'ULTRIUM-TD4', 'statistics_path': '/tmp/enstore/enstore/LTO4_48.stat', 'single_filemark': 1, 'mount_delay': 15, 'mover_address': ('131.225.13.176', 7848), 'check_first_written_file': 0, 'callback_addr': ('131.225.13.176', 59187), 'data_ip': 'stkenmvr148a', 'logname': 'LTO4_48MV', 'hostip': '131.225.13.176', 'do_cleaning': 'No', 'media_changer': 'SL8500G1.media_changer', 'min_buffer': 67108864L}, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fs/usr/.(access)(000F0000000000000A9848A0)', 'uid': 0, 'uname': 'root', 'rmajor': 0, 'machine': ('Linux', 'stkendca23a.fnal.gov', '2.6.9-89.0.9.ELsmp', '#1 SMP Mon Aug 24 13:47:43 CDT 2009', 'i686'), 'sanity_size': 65536, 'pstat': (33188, 261638304L, 20L, 1, 42411, 5111, 127633279L, 1255146250, 1250454867, 1250451128), 'mode_octal': '0100744', 'mode': 33252, 'size_bytes': 127633279L, 'gname': 'root', 'gid': 0, 'fullname': '/diskb/read-pool-2/data/000F0000000000000A9848A0', 'inode': 261638304L, 'minor': 20}, 'version': 'v3_8b CVS $Revision: 1.922 $ ', 'encp_daq': None, 'client_crc': 1, 'infile': '/pnfs/fs/usr/.(access)(000F0000000000000A983D10)/.(access)(000F0000000000000A9848A0)', 'status': ('ok', None), 'volume': 'VON584', 'outfile': '/diskb/read-pool-2/data/000F0000000000000A9848A0', 'fc': {'status': ('ok', None), 'pnfsid': '000F0000000000000A9848A0', 'pnfs_name0': '/pnfs/fnal.gov/usr/minos/reco_far/dogwood1/.bntp_data/2009-05/F00043546_0000.spill.bntp.dogwood1.0.root', 'deleted': 'no', 'external_label': 'VON584', 'drive': 'stkenmvr146a:/dev/rmt/tps0d0n:9311032672', 'update': '2009-08-16 15:34:27.42089', 'gid': 0, 'location_cookie': '0000_000000000_0000557', 'complete_crc': 3824704097L, 'address': ('131.225.13.59', 7501), 'uid': 0, 'bfid': 'CDMS125045486600000', 'sanity_cookie': (65536L, 288456852L), 'size': 127633279L}, 'infilepath': '/pnfs/fs/usr/.(access)(000F0000000000000A9848A0)', 'at_the_top': 2, 'bfid': 'CDMS125045486600000', 'local_inode': 1073745108L, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.13.104', 43614), 'work': 'read_from_hsm', 'override_ro_mount': 0, 'times': {'lm_dequeued': 1257141638.781302, 'encp_start_time': 1257141638.1616769, 'job_queued': 1257141638.712321, 't0': 1257141638, 'in_queue': 0.068939924240112305}, 'user_level': 'user2', 'unique_id': 'stkendca23a.fnal.gov-1257141638-23854-0'} Thread client_connect_thread 00:00:38 stkendca23a.fnal.gov 023854 root I ENCP Control socket ('131.225.13.104', 43614) is connected to ('131.225.13.176', 59188) for stkendca23a.fnal.gov-1257141638-23854-0. elapsed=0.680sec 00:00:38 stkendca23a.fnal.gov 023854 root I ENCP Received callback ticket from mover LTO4_48.mover for transfer stkendca23a.fnal.gov-1257141638-23854-0. elapsed=0.680sec 00:00:38 stkendca23a.fnal.gov 023854 root I ENCP Atempting to connect data socket to mover at ('131.225.13.176', 59187). 00:00:38 stkendca23a.fnal.gov 023854 root I ENCP Connecting data socket to mover (('131.225.13.176', 59187)) with interface 131.225.13.104. elapsed=0.680sec 00:00:38 stkendca23a.fnal.gov 023854 root I ENCP Data socket ('131.225.13.104', 43615) is connected to ('131.225.13.176', 59187). elapsed=0.680sec 00:00:38 stkendca23a.fnal.gov 023854 root I ENCP Opening local file. 00:00:38 stkenmvr148a.fnal.gov 006010 root I LTO4_48MV MSG_TYPE=MC_LOAD_REQ mounting VON584 ULTRIUM-TD4 00:00:38 Thread media_thread 00:00:38 stkensrv4n.fnal.gov 003689 enstore I SL8500G1MC MC Overflow: 4 mount VON584 0,5,1,15 00:00:38 stkenmvr148a.fnal.gov 006010 root E LTO4_48MV loadvol ('MEDIA_CHANGER_QUEUE_FULL', 0, 'retry later') Thread media_thread 00:00:38 stkenmvr148a.fnal.gov 006010 root I LTO4_48MV Media Changer returned MEDIA_CHANGER_QUEUE_FULL Thread media_thread 00:00:39 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:39 stkenmvr208a.fnal.gov 006111 root I LTO4_108MV written bytes 2147475968/2147475968, blocks 16384 header 190 trailer 322 Thread tape_thread 00:00:39 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:40 stkensrv4n.fnal.gov 003689 enstore I SL8500G1MC MC Overflow: 4 mount VOR381 0,5,1,3 00:00:40 stkenmvr149a.fnal.gov 006318 root E LTO4_49MV loadvol ('MEDIA_CHANGER_QUEUE_FULL', 0, 'retry later') Thread media_thread 00:00:40 stkenmvr149a.fnal.gov 006318 root I LTO4_49MV Media Changer returned MEDIA_CHANGER_QUEUE_FULL Thread media_thread 00:00:40 stkenmvr203a.fnal.gov 006116 root I LTO4_103MV skip_fm 37 Thread seek_thread 00:00:40 stkenmvr203a.fnal.gov 006116 root I LTO4_103MV write_tape starting, bytes_to_write=2107477504 Thread tape_thread 00:00:40 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM 00:00:40 stkenmvr203a.fnal.gov 006116 root I LTO4_103MV Write starting. Tape VOR423 absolute location in blocks 898210 Thread tape_thread 00:00:41 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:41 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:41 stkenmvr208a.fnal.gov 006111 root I LTO4_108MV filemarks written. Tape VON217 absolute location in blocks 4938843 Thread tape_thread 00:00:41 stkenmvr208a.fnal.gov 006111 root I LTO4_108MV write_tape timing:pull 0.213254451752 write 7.34478449821 crc_check 9.90359258652 freespace 0.16535115242 block_write 17.6269826889 idle 0.293635845184 Thread tape_thread 00:00:41 stkenmvr208a.fnal.gov 006111 root I LTO4_108MV drive stats after write. Tape VON217 position 323 block 0 block_size 0 bloc_loc 4938843 tot_blocks 762939000 BOT 0 read_err 0 write_err 0 bytes 2147475968 block_write_tot 17.6269826889 tape_rate 121828903.216 Thread tape_thread 00:00:41 stkenmvr208a.fnal.gov 006111 root I LTO4_108MV remaining bytes info in DB 170174464000 estimated 168026988032 reported from drive 168145920000 position 324 Thread tape_thread 00:00:41 stkenmvr208a.fnal.gov 006111 root I LTO4_108MV new bitfile request {'uid': 0, 'external_label': 'VON217', 'gid': 0, 'location_cookie': '0000_000000000_0000323', 'complete_crc': 2616781829L, 'sanity_cookie': (65536L, 698187649L), 'size': 2147475456L} Thread tape_thread 00:00:41 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:41 stkenmvr208a.fnal.gov 006111 root I LTO4_108MV set remaining: VON217 168145920000 0000_000000000_0000324 Thread tape_thread 00:00:41 last message repeated 1 times 00:00:41 stkensrv0n.fnal.gov 028762 enstore I VOLSRV Updating update volume set last_access = '2009-11-02 00:00:41', eod_cookie = '0000_000000000_0000324', non_del_files = '353', remaining_bytes = 168145920000 where label = 'VON217'; 00:00:41 stkenmvr208a.fnal.gov 006111 root I LTO4_108MV transfer complete volume=VON217 location=324 Thread tape_thread 00:00:41 last message repeated 1 times 00:00:41 stkensrv0n.fnal.gov 028762 enstore I VOLSRV Updating update volume set sum_wr_access = '323', modification_time = '2009-11-02 00:00:41' where label = 'VON217'; 00:00:41 stkenmvr208a.fnal.gov 006111 root I LTO4_108MV Sending done to client: {'vc': {'comment': '', 'storage_group': 'exp-db', 'blocksize': 131072, 'declared': 1240352603.0, 'library': 'CD-LTO4G1', 'si_time': [0.0, 0.0], 'user_inhibit': ['none', 'none'], 'system_inhibit': ['none', 'none'], 'external_label': 'VON217', 'wrapper': 'cpio_odc', 'remaining_bytes': 170174464000L, 'sum_mounts': 62, 'sum_rd_access': 0, 'file_family_width': 2, 'last_access': 1257141573.0, 'media_type': 'LTO4', 'status': ('ok', None), 'eod_cookie': '0000_000000000_0000323', 'capacity_bytes': 858993459200L, 'non_del_files': 352, 'sum_wr_err': 0, 'sum_wr_access': 322, 'file_family': 'monthly-cdf-offline', 'address': ('131.225.13.59', 7502), 'volume_family': 'exp-db.monthly-cdf-offline.cpio_odc', 'modification_time': 1257141573.0, 'write_protected': 'n', 'sum_rd_err': 0, 'first_access': 1241268501.0}, 'outfilepath': '/pnfs/fnal.gov/usr/exp-db/monthly/cdf-offline/cdfofpr2/2009/11-November/01/channel01/CDFOFPR2_t701800607_s30271_p4', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 1, 'delayed_dismount': None, 'curpri': 1}, 'file_size': 2147475456L, 'mover': {'max_buffer': 2621440000L, 'check_written_file': 600, 'max_failures': 5, 'library': 'CD-LTO4G1.library_manager', 'local_mover': 0, 'single_filemark': 1, 'max_consecutive_failures': 3, 'norestart': 'INQ', 'port': 7868, 'mc_device': '0,5,1,11', 'compression': 0, 'serial_num': '1310154353', 'update_interval_in_bound': 2, 'update_interval': 5, 'syslog_entry': 'sense|st[0-9]', 'status': ('ok', None), 'max_rate': 73400320.0, 'vendor_id': 'IBM', 'driver': 'FTTDriver', 'host': 'stkenmvr208a', 'max_time_in_state': 1200, 'device': 'stkenmvr208a:/dev/rmt/tps0d0n', 'name': 'LTO4_108.mover', 'send_stats': 1, 'product_id': 'ULTRIUM-TD4', 'statistics_path': '/tmp/enstore/enstore/LTO4_108.stat', 'mover_address': ('131.225.13.204', 7868), 'mount_delay': 15, 'check_first_written_file': 0, 'callback_addr': ('131.225.13.204', 57950), 'data_ip': 'stkenmvr208a', 'logname': 'LTO4_108MV', 'hostip': '131.225.13.204', 'do_cleaning': 'No', 'media_changer': 'SL8500G1.media_changer'}, 'ignore_fair_share': None, 'retry': None, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fnal.gov/usr/exp-db/monthly/cdf-offline/cdfofpr2/2009/11-November/01/channel01/CDFOFPR2_t701800607_s30271_p4', 'uid': 0, 'uname': 'root', 'mtime': 1257141043, 'rmajor': 0, 'machine': ('Linux', 'stkendca20a.fnal.gov', '2.6.9-89.0.9.ELsmp', '#1 SMP Mon Aug 24 13:47:43 CDT 2009', 'i686'), 'sanity_size': 65536, 'type': 'cpio_odc', 'pstat': (16877, 820663368L, 20L, 1, 1602, 2752, 512L, 1257130346, 1257130346, 1254398469), 'mode_octal': '0100016', 'mode': 33188, 'size_bytes': 2147475456L, 'gname': 'root', 'gid': 0, 'fullname': '/diskb/write-pool-2/data/003000000000000006199E08', 'inode': 806985224L, 'minor': 20}, 'version': 'v3_8b CVS $Revision: 1.922 $ ', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/diskb/write-pool-2/data/003000000000000006199E08', 'status': ('ok', None), 'outfile': '/pnfs/fnal.gov/usr/exp-db/monthly/cdf-offline/cdfofpr2/2009/11-November/01/channel01/.(access)(003000000000000006199E08)', 'fc': {'uid': 0, 'external_label': 'VON217', 'gid': 0, 'location_cookie': '0000_000000000_0000323', 'complete_crc': 2616781829L, 'bfid': 'CDMS125714164100000', 'sanity_cookie': (65536L, 698187649L), 'size': 2147475456L}, 'infilepath': '/diskb/write-pool-2/data/003000000000000006199E08', 'at_the_top': 15, 'local_inode': 268812465L, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.13.12', 57124), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'encp_start_time': 1257141040.375174, 'seek_time': 0.029410123825073242, 'transfer_time': 67.449209928512573, 't0': 1257141040, 'mount_time': 0.014987945556640625, 'lm_dequeued': 1257141574.060482, 'job_queued': 1257141043.4853759, 'drive_call_time': 7.1252551078796387, 'in_queue': 530.57482814788818, 'drive_transfer_time': 20.341939687728882}, 'user_level': 'user2', 'unique_id': 'stkendca20a.fnal.gov-(trunc. 4100) Thread tape_thread 00:00:41 stkenmvr208a.fnal.gov 006111 root I LTO4_108MV Done is sent Thread tape_thread 00:00:41 stkenmvr208a.fnal.gov 006111 root I LTO4_108MV maybe_clean Thread tape_thread 00:00:41 stkenmvr208a.fnal.gov 006111 root I LTO4_108MV returned from maybe_clean Thread tape_thread 00:00:41 stkendca20a.fnal.gov 032266 root I ENCP Received final dialog for stkendca20a.fnal.gov-1257141043-32266-0. elapsed=601.270sec 00:00:41 stkendca20a.fnal.gov 032266 root I ENCP leaving transfer_file(). 00:00:41 stkendca20a.fnal.gov 032266 root I ENCP pf_status = ('ok', None) skip_layer_cleanup = False 00:00:42 stkensrv0n.fnal.gov 028796 enstore I FILSRV Updating update file set deleted = 'n', pnfs_path = '/pnfs/fnal.gov/usr/exp-db/monthly/cdf-offline/cdfofpr2/2009/11-November/01/channel01/CDFOFPR2_t701800607_s30271_p4', drive = 'stkenmvr208a:/dev/rmt/tps0d0n:1310154353', pnfs_id = '003000000000000006199E08' where bfid = 'CDMS125714164100000'; 00:00:42 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:42 stkendca20a.fnal.gov 032266 root I ENCP MSG_TYPE=ENCP_XFER write_to_hsm /diskb/write-pool-2/data/003000000000000006199E08 -> /pnfs/fnal.gov/usr/exp-db/monthly/cdf-offline/cdfofpr2/2009/11-November/01/channel01/CDFOFPR2_t701800607_s30271_p4: 2147475456 bytes copied to VON217 at 30.4 MB/S (57.3 MB/S network) (101 MB/S drive) (32 MB/S disk) mover=LTO4_108.mover drive_id=ULTRIUM-TD4 drive_sn=1310154353 drive_vendor=IBM elapsed=601.78 {'storage_group': 'exp-db', 'encp_crc': 2616781829L, 'disk_rate': 32.037802678626392, 'drive_rate': 100.67831381564048, 'driver': 'FTTDriver', 'transfer_rate': 30.400486945630433, 'mover_interface': 'stkenmvr208a', 'overall_rate': 30.112427642585025, 'media_changer': 'SL8500G1.media_changer', 'network_rate': 30.400486945630433, 'encp_ip': '131.225.13.12', 'unique_id': 'stkendca20a.fnal.gov-1257141043-32266-0'} 00:00:42 stkendca20a.fnal.gov 032266 root I ENCP File /pnfs/fnal.gov/usr/exp-db/monthly/cdf-offline/cdfofpr2/2009/11-November/01/channel01/CDFOFPR2_t701800607_s30271_p4 copied successfully. 00:00:42 stkendca20a.fnal.gov 032266 root I ENCP Completed transferring 2147475456 bytes in 1 files in 601.779540062 sec. Overall rate = 30.1 MB/sec. Transfer rate = 30.4 MB/sec. Network rate = 57.3 MB/sec. Drive rate = 101 MB/sec. Disk rate = 32 MB/sec. Exit status = 0. 00:00:42 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM write_to_hsm next work on vol=VON217 mover=LTO4_108.mover requester:root 00:00:42 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM HAVE_BOUND:sending write_to_hsm /pnfs/fnal.gov/usr/exp-db/monthly/cdf-offline/cdfofpr2/2009/11-November/01/channel01/CDFOFPR2_t701802222_s30285_p1 to mover LTO4_108.mover ('131.225.13.204', 7868) DEL_DISM None 00:00:42 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM MSG_TYPE=ADD_TO_LMQ request added to work at movers queue for storage group : exp-db 00:00:42 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM HAVE_BOUND:sending {'vc': {'storage_group': 'exp-db', 'external_label': 'VON217', 'library': 'CD-LTO4G1', 'file_family': 'monthly-cdf-offline', 'wrapper': 'cpio_odc', 'address': ('131.225.13.59', 7502), 'volume_family': 'exp-db.monthly-cdf-offline.cpio_odc', 'file_family_width': 2}, 'outfilepath': '/pnfs/fnal.gov/usr/exp-db/monthly/cdf-offline/cdfofpr2/2009/11-November/01/channel01/CDFOFPR2_t701802222_s30285_p1', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 1, 'delayed_dismount': None, 'curpri': 1}, 'file_size': 2147475456L, 'mover': 'LTO4_108.mover', 'ignore_fair_share': None, 'retry': None, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fnal.gov/usr/exp-db/monthly/cdf-offline/cdfofpr2/2009/11-November/01/channel01/CDFOFPR2_t701802222_s30285_p1', 'uid': 0, 'uname': 'root', 'type': 'cpio_odc', 'gid': 0, 'mtime': 1257141044, 'rmajor': 0, 'machine': ('Linux', 'stkendca20a.fnal.gov', '2.6.9-89.0.9.ELsmp', '#1 SMP Mon Aug 24 13:47:43 CDT 2009', 'i686'), 'sanity_size': 65536, 'pstat': (16877, 820663368L, 20L, 1, 1602, 2752, 512L, 1257130346, 1257130346, 1254398469), 'mode_octal': '0100016', 'mode': 33188, 'gname': 'root', 'size_bytes': 2147475456L, 'fullname': '/diskb/write-pool-2/data/00300000000000000619AFC8', 'inode': 806989768L, 'minor': 20}, 'version': 'v3_8b CVS $Revision: 1.922 $ ', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/diskb/write-pool-2/data/00300000000000000619AFC8', 'status': ('ok', None), 'outfile': '/pnfs/fnal.gov/usr/exp-db/monthly/cdf-offline/cdfofpr2/2009/11-November/01/channel01/.(access)(00300000000000000619AFC8)', 'fc': {'size': 2147475456L, 'external_label': 'VON217', 'address': ('131.225.13.59', 7501)}, 'infilepath': '/diskb/write-pool-2/data/00300000000000000619AFC8', 'at_the_top': 52, 'local_inode': 271282418L, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.13.12', 57126), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'lm_dequeued': 1257141642.189368, 'encp_start_time': 1257141041.831778, 'job_queued': 1257141044.8637171, 't0': 1257141041, 'in_queue': 597.32538795471191}, 'user_level': 'user2', 'unique_id': 'stkendca20a.fnal.gov-1257141044-32249-0'} to mover 00:00:42 last message repeated 1 times 00:00:42 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:42 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:42 stkenmvr208a.fnal.gov 006111 root I LTO4_108MV WRITE_TO_HSM Thread MainThread 00:00:42 stkenmvr208a.fnal.gov 006111 root I LTO4_108MV SENDING {'vc': {'storage_group': 'exp-db', 'external_label': 'VON217', 'library': 'CD-LTO4G1', 'file_family': 'monthly-cdf-offline', 'wrapper': 'cpio_odc', 'address': ('131.225.13.59', 7502), 'volume_family': 'exp-db.monthly-cdf-offline.cpio_odc', 'file_family_width': 2}, 'outfilepath': '/pnfs/fnal.gov/usr/exp-db/monthly/cdf-offline/cdfofpr2/2009/11-November/01/channel01/CDFOFPR2_t701802222_s30285_p1', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 1, 'delayed_dismount': None, 'curpri': 1}, 'file_size': 2147475456L, 'mover': {'max_buffer': 2621440000L, 'check_written_file': 600, 'max_failures': 5, 'library': 'CD-LTO4G1.library_manager', 'local_mover': 0, 'single_filemark': 1, 'max_consecutive_failures': 3, 'norestart': 'INQ', 'port': 7868, 'mc_device': '0,5,1,11', 'compression': 0, 'serial_num': '1310154353', 'update_interval_in_bound': 2, 'update_interval': 5, 'syslog_entry': 'sense|st[0-9]', 'status': ('ok', None), 'max_rate': 73400320.0, 'vendor_id': 'IBM', 'driver': 'FTTDriver', 'host': 'stkenmvr208a', 'max_time_in_state': 1200, 'device': 'stkenmvr208a:/dev/rmt/tps0d0n', 'name': 'LTO4_108.mover', 'send_stats': 1, 'product_id': 'ULTRIUM-TD4', 'statistics_path': '/tmp/enstore/enstore/LTO4_108.stat', 'mover_address': ('131.225.13.204', 7868), 'mount_delay': 15, 'check_first_written_file': 0, 'callback_addr': ('131.225.13.204', 57957), 'data_ip': 'stkenmvr208a', 'logname': 'LTO4_108MV', 'hostip': '131.225.13.204', 'do_cleaning': 'No', 'media_changer': 'SL8500G1.media_changer'}, 'ignore_fair_share': None, 'retry': None, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fnal.gov/usr/exp-db/monthly/cdf-offline/cdfofpr2/2009/11-November/01/channel01/CDFOFPR2_t701802222_s30285_p1', 'uid': 0, 'uname': 'root', 'mtime': 1257141044, 'rmajor': 0, 'machine': ('Linux', 'stkendca20a.fnal.gov', '2.6.9-89.0.9.ELsmp', '#1 SMP Mon Aug 24 13:47:43 CDT 2009', 'i686'), 'sanity_size': 65536, 'type': 'cpio_odc', 'pstat': (16877, 820663368L, 20L, 1, 1602, 2752, 512L, 1257130346, 1257130346, 1254398469), 'mode_octal': '0100016', 'mode': 33188, 'size_bytes': 2147475456L, 'gname': 'root', 'gid': 0, 'fullname': '/diskb/write-pool-2/data/00300000000000000619AFC8', 'inode': 806989768L, 'minor': 20}, 'version': 'v3_8b CVS $Revision: 1.922 $ ', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/diskb/write-pool-2/data/00300000000000000619AFC8', 'status': ('ok', None), 'outfile': '/pnfs/fnal.gov/usr/exp-db/monthly/cdf-offline/cdfofpr2/2009/11-November/01/channel01/.(access)(00300000000000000619AFC8)', 'fc': {'size': 2147475456L, 'external_label': 'VON217', 'address': ('131.225.13.59', 7501)}, 'infilepath': '/diskb/write-pool-2/data/00300000000000000619AFC8', 'at_the_top': 52, 'local_inode': 271282418L, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.13.12', 57126), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'lm_dequeued': 1257141642.189368, 'encp_start_time': 1257141041.831778, 'job_queued': 1257141044.8637171, 't0': 1257141041, 'in_queue': 597.32538795471191}, 'user_level': 'user2', 'unique_id': 'stkendca20a.fnal.gov-1257141044-32249-0'} Thread client_connect_thread 00:00:42 stkendca20a.fnal.gov 032249 root I ENCP Control socket ('131.225.13.12', 57126) is connected to ('131.225.13.204', 57958) for stkendca20a.fnal.gov-1257141044-32249-0. elapsed=601.050sec 00:00:42 stkendca20a.fnal.gov 032249 root I ENCP Received callback ticket from mover LTO4_108.mover for transfer stkendca20a.fnal.gov-1257141044-32249-0. elapsed=601.060sec 00:00:42 stkendca20a.fnal.gov 032249 root I ENCP Atempting to connect data socket to mover at ('131.225.13.204', 57957). 00:00:42 stkendca20a.fnal.gov 032249 root I ENCP Connecting data socket to mover (('131.225.13.204', 57957)) with interface 131.225.13.12. elapsed=601.060sec 00:00:42 stkendca20a.fnal.gov 032249 root I ENCP Data socket ('131.225.13.12', 57173) is connected to ('131.225.13.204', 57957). elapsed=601.060sec 00:00:42 stkendca20a.fnal.gov 032249 root I ENCP Opening local file. 00:00:42 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM 00:00:42 stkenmvr208a.fnal.gov 006111 root I LTO4_108MV tape position: target 324 current 324 0 Thread seek_thread 00:00:42 stkenmvr208a.fnal.gov 006111 root I LTO4_108MV write_tape starting, bytes_to_write=2147475968 Thread tape_thread 00:00:42 stkenmvr208a.fnal.gov 006111 root I LTO4_108MV Write starting. Tape VON217 absolute location in blocks 4938843 Thread tape_thread 00:00:42 stkendca20a.fnal.gov 032249 root I ENCP pf_status = ('ok', None) skip_layer_cleanup = False 00:00:42 stkenmvr214a.fnal.gov 006260 root I LTO4_114MV Tape is ejected Thread media_thread 00:00:42 stkenmvr214a.fnal.gov 006260 root I LTO4_114MV dismounting VOO096 Thread media_thread 00:00:42 stkensrv4n.fnal.gov 005843 enstore I SL8500F1MC PROCESSING dismount VOO096 1,1,1,8 00:00:42 stkensrv4n.fnal.gov 003689 enstore I SL8500G1MC MC Overflow: 4 mount VOP202 0,5,1,12 00:00:42 stkenmvr151a.fnal.gov 013228 root E LTO4_51MV loadvol ('MEDIA_CHANGER_QUEUE_FULL', 0, 'retry later') Thread media_thread 00:00:42 stkenmvr151a.fnal.gov 013228 root I LTO4_51MV Media Changer returned MEDIA_CHANGER_QUEUE_FULL Thread media_thread 00:00:43 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:43 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:43 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:43 stkensrv4n.fnal.gov 029399 enstore I SL8500G1MC mount VOR416 0,7,1,0 => 0,Mount: VOR416 mounted on 0, 7, 1, 0 00:00:43 stkensrv4n.fnal.gov 003689 enstore I SL8500G1MC FINISHED mount VOR416 0,7,1,0 returned ('ok', 0, 'mount VOR416 0,7,1,0 => 0,Mount: VOR416 mounted on 0, 7, 1, 0', '', '') 00:00:43 stkensrv4n.fnal.gov 003689 enstore I SL8500G1MC PROCESSING dismount VOR385 0,6,1,3 00:00:43 stkensrv0n.fnal.gov 028762 enstore I VOLSRV Updating update volume set sum_mounts = '85', last_access = '2009-11-02 00:00:43' where label = 'VOR416'; 00:00:43 stkenmvr143a.fnal.gov 028425 root I LTO4_43MV MSG_TYPE=MC_LOAD_DONE mounted VOR416 ULTRIUM-TD4 00:00:43 Thread media_thread 00:00:43 stkenmvr146a.fnal.gov 005990 root I LTO4_46MV skip_fm -2 Thread tape_thread 00:00:43 stkenmvr146a.fnal.gov 005990 root I LTO4_46MV skip_fm 1 Thread tape_thread 00:00:44 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM 00:00:45 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:45 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:45 stkensrv4n.fnal.gov 005357 enstore E LTO4G1LM MOVER ERROR RQ {'status': ('ok', None), 'returned_work': None, 'library': None, 'mover_type': '', 'current_location': '0000_000000000_0000000', 'ip_map': '', 'operation': None, 'read_only': 0, 'current_time': 1257141645.2945859, 'error_source': None, 'external_label': None, 'state': 'OFFLINE', 'r_a': (('131.225.13.194', 54880), 11095L, '131.225.13.194-54880-1256586929.509013-4866'), 'library_list': [('CD-LTO4G1.library_manager', ('131.225.13.4', 8541))], 'current_priority': (None, None), 'transfer_deficiency': 1, 'time_in_state': 554714.93452787399, 'address': ('131.225.13.194', 7860), 'volume_family': None, 'work': 'mover_error', 'mover': 'LTO4_99.mover', 'volume_status': (['none', 'none'], ['none', 'none']), 'unique_id': None} 00:00:45 stkensrv4n.fnal.gov 030262 enstore I SL8500G1MC query drive 0,6,1,3 => 0,0,6,1,3 online in use VOR385 IBM-LTO4 00:00:45 stkensrv4n.fnal.gov 003689 enstore I SL8500G1MC MC Overflow: 4 mount VOR198 0,5,1,7 00:00:45 stkenmvr209a.fnal.gov 006118 root E LTO4_109MV loadvol ('MEDIA_CHANGER_QUEUE_FULL', 0, 'retry later') Thread media_thread 00:00:45 stkenmvr209a.fnal.gov 006118 root I LTO4_109MV Media Changer returned MEDIA_CHANGER_QUEUE_FULL Thread media_thread 00:00:46 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM 00:00:47 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:47 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:47 stkensrv4n.fnal.gov 003689 enstore I SL8500G1MC MC Overflow: 4 dismount VOR424 0,6,1,11 00:00:47 stkenmvr200a.fnal.gov 005076 root E LTO4_100MV unloadvol ('MEDIA_CHANGER_QUEUE_FULL', 0, 'retry later') Thread media_thread 00:00:47 stkenmvr200a.fnal.gov 005076 root I LTO4_100MV Media Changer returned MEDIA_CHANGER_QUEUE_FULL Thread media_thread 00:00:48 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM 00:00:48 stkensrv4n.fnal.gov 003689 enstore I SL8500G1MC MC Overflow: 4 mount VOR382 0,6,1,13 00:00:48 stkenmvr204a.fnal.gov 006261 root E LTO4_104MV loadvol ('MEDIA_CHANGER_QUEUE_FULL', 0, 'retry later') Thread media_thread 00:00:48 stkenmvr204a.fnal.gov 006261 root I LTO4_104MV Media Changer returned MEDIA_CHANGER_QUEUE_FULL Thread media_thread 00:00:48 stkensrv4n.fnal.gov 003689 enstore I SL8500G1MC MC Overflow: 4 mount VON584 0,5,1,15 00:00:48 stkenmvr148a.fnal.gov 006010 root E LTO4_48MV loadvol ('MEDIA_CHANGER_QUEUE_FULL', 0, 'retry later') Thread media_thread 00:00:48 stkenmvr148a.fnal.gov 006010 root I LTO4_48MV Media Changer returned MEDIA_CHANGER_QUEUE_FULL Thread media_thread 00:00:48 stkensrv4n.fnal.gov 030259 enstore I SL8500F1MC query drive 1,1,1,8 => 0,1,1,1,8 online in use VOO096 IBM-LTO4 00:00:49 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:49 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:50 stkensrv4n.fnal.gov 003689 enstore I SL8500G1MC MC Overflow: 4 mount VOR381 0,5,1,3 00:00:50 stkenmvr149a.fnal.gov 006318 root E LTO4_49MV loadvol ('MEDIA_CHANGER_QUEUE_FULL', 0, 'retry later') Thread media_thread 00:00:50 stkenmvr149a.fnal.gov 006318 root I LTO4_49MV Media Changer returned MEDIA_CHANGER_QUEUE_FULL Thread media_thread 00:00:50 stkensrv2n.fnal.gov 006421 enstore E INQSRV Could not create seen-down file lock file (/srv2/enstore/www/web-pages//enstore_seen_down.py.lock) 00:00:50 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM 00:00:50 stkensrv2n.fnal.gov 006421 enstore E INQSRV Could not read file /srv2/enstore/www/web-pages//enstore_seen_down.py 00:00:51 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:51 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:52 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM 00:00:52 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:52 stkensrv4n.fnal.gov 003689 enstore I SL8500G1MC MC Overflow: 4 mount VOP202 0,5,1,12 00:00:52 stkenmvr151a.fnal.gov 013228 root E LTO4_51MV loadvol ('MEDIA_CHANGER_QUEUE_FULL', 0, 'retry later') Thread media_thread 00:00:52 stkenmvr151a.fnal.gov 013228 root I LTO4_51MV Media Changer returned MEDIA_CHANGER_QUEUE_FULL Thread media_thread 00:00:52 stkensrv2n.fnal.gov 006421 enstore E INQSRV Could not create outage file lock file (/srv2/enstore/www/web-pages//enstore_outage.py.lock) 00:00:53 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:53 stkenmvr140a.fnal.gov 030526 root I LTO4_40MV Updating stats Thread media_thread 00:00:53 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:53 stkenmvr140a.fnal.gov 030526 root I LTO4_40MV writing a tape mark before dismount Thread media_thread 00:00:53 stkenmvr207a.fnal.gov 006260 root I LTO4_107MV Tape is ejected Thread media_thread 00:00:53 stkenmvr207a.fnal.gov 006260 root I LTO4_107MV dismounting VON399 Thread media_thread 00:00:53 stkensrv2n.fnal.gov 006421 enstore E INQSRV Could not read file /srv2/enstore/www/web-pages//enstore_outage.py 00:00:53 stkensrv4n.fnal.gov 003689 enstore I SL8500G1MC MC Overflow: 4 dismount VON399 0,6,1,4 00:00:53 stkenmvr207a.fnal.gov 006260 root E LTO4_107MV unloadvol ('MEDIA_CHANGER_QUEUE_FULL', 0, 'retry later') Thread media_thread 00:00:53 stkenmvr207a.fnal.gov 006260 root I LTO4_107MV Media Changer returned MEDIA_CHANGER_QUEUE_FULL Thread media_thread 00:00:53 stkensrv4n.fnal.gov 029800 enstore I SL8500G1MC dismount VOLUME 0,6,1,1 force => 0,Dismount: Forced dismount of VOR422 from 0, 6, 1, 1 00:00:53 stkensrv4n.fnal.gov 003689 enstore I SL8500G1MC FINISHED dismount VOR422 0,6,1,1 returned ('ok', 0, 'dismount VOLUME 0,6,1,1 force => 0,Dismount: Forced dismount of VOR422 from 0, 6, 1, 1', '', '') 00:00:53 stkenmvr205a.fnal.gov 011191 root I LTO4_105MV dismounted VOR422 ULTRIUM-TD4 00:00:53 Thread media_thread 00:00:54 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM 00:00:55 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:55 stkensrv4n.fnal.gov 003689 enstore I SL8500G1MC PROCESSING mount VOR198 0,5,1,7 00:00:55 stkensrv4n.fnal.gov 029788 enstore I SL8500G1MC mount VOR407 0,5,1,14 => 0,Mount: VOR407 mounted on 0, 5, 1,14 00:00:55 stkensrv4n.fnal.gov 003689 enstore I SL8500G1MC FINISHED mount VOR407 0,5,1,14 returned ('ok', 0, 'mount VOR407 0,5,1,14 => 0,Mount: VOR407 mounted on 0, 5, 1,14', '', '') 00:00:55 last message repeated 1 times 00:00:55 stkensrv0n.fnal.gov 028762 enstore I VOLSRV Updating update volume set sum_mounts = '111', last_access = '2009-11-02 00:00:55' where label = 'VOR407'; 00:00:55 stkenmvr210a.fnal.gov 011436 root I LTO4_110MV MSG_TYPE=MC_LOAD_DONE mounted VOR407 ULTRIUM-TD4 00:00:55 Thread media_thread 00:00:56 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:56 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM 00:00:56 stkenmvr140a.fnal.gov 030526 root I LTO4_40MV Ejecting tape Thread media_thread 00:00:57 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:57 stkensrv4n.fnal.gov 003689 enstore I SL8500G1MC PROCESSING dismount VOR424 0,6,1,11 00:00:58 cmsstor139.fnal.gov 005777 root I ENCP File /storage/data3/write-pool-3/data/0007000000000000618C8900 transfered for cmsstor139.fnal.gov-1257141501-5777-0 in 17.8086948395 seconds. 00:00:58 stkensrv4n.fnal.gov 030269 enstore I SL8500G1MC query drive 0,5,1,7 => 0,0,5,1,7 online available IBM-LTO4 00:00:58 stkensrv4n.fnal.gov 030269 enstore I SL8500G1MC Dismount 14 ignored: Drive 0,5,1,7 is empty. Thought VOR198 was there =>. dismount VOLUME 0,5,1,7 force => ok,0,5,1,7 online available IBM-LTO4 00:00:58 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM 00:00:58 stkensrv4n.fnal.gov 003689 enstore I SL8500G1MC MC Overflow: 4 mount VOR382 0,6,1,13 00:00:58 stkenmvr204a.fnal.gov 006261 root E LTO4_104MV loadvol ('MEDIA_CHANGER_QUEUE_FULL', 0, 'retry later') Thread media_thread 00:00:58 stkenmvr204a.fnal.gov 006261 root I LTO4_104MV Media Changer returned MEDIA_CHANGER_QUEUE_FULL Thread media_thread 00:00:58 stkensrv4n.fnal.gov 003689 enstore I SL8500G1MC MC Overflow: 4 mount VON584 0,5,1,15 00:00:58 stkenmvr148a.fnal.gov 006010 root E LTO4_48MV loadvol ('MEDIA_CHANGER_QUEUE_FULL', 0, 'retry later') Thread media_thread 00:00:58 stkenmvr148a.fnal.gov 006010 root I LTO4_48MV Media Changer returned MEDIA_CHANGER_QUEUE_FULL Thread media_thread 00:00:58 stkendca22a.fnal.gov 022720 root I ENCP Version: v3_8b CVS $Revision: 1.922 $ OS: Linux 2.6.9-89.0.9.ELsmp i686 Release: Scientific Linux Fermi LTS release 4.4 (Wilson) User: root(0) Group: root(0) Euser: root(0) Egroup: root(0) Library: garbage Storage Group: test File Family: dcache FF Wrapper: cpio_odc FF Width: 1 Current working directory: stkendca22a.fnal.gov:/home/enstore/dcache-log Command line: /fnal/ups/prd/encp/v3_8b/Linux-2-6-dcache/encp --verbose 4 --threaded --ecrc --bypass-filesystem-max-filesize-check --pnfs-mount /pnfs/fs --shortcut --override-path /pnfs/fnal.gov/usr/fermigrid/volatile/fermilab/timur-test/broken-lib/timur-test-0003 --put-cache 0005000000000000005FA5F8 /diskb/write-pool-2/data/0005000000000000005FA5F8 00:00:59 stkendca22a.fnal.gov 022758 root I ENCP Version: v3_8b CVS $Revision: 1.922 $ OS: Linux 2.6.9-89.0.9.ELsmp i686 Release: Scientific Linux Fermi LTS release 4.4 (Wilson) User: root(0) Group: root(0) Euser: root(0) Egroup: root(0) Library: garbage Storage Group: test File Family: dcache FF Wrapper: cpio_odc FF Width: 1 Current working directory: stkendca22a.fnal.gov:/home/enstore/dcache-log Command line: /fnal/ups/prd/encp/v3_8b/Linux-2-6-dcache/encp --verbose 4 --threaded --ecrc --bypass-filesystem-max-filesize-check --pnfs-mount /pnfs/fs --shortcut --override-path /pnfs/fnal.gov/usr/fermigrid/volatile/fermilab/timur-test/broken-lib/timur-test-0001 --put-cache 0005000000000000005FA590 /diskb/write-pool-2/data/0005000000000000005FA590 00:00:59 stkendca22a.fnal.gov 022776 root I ENCP Version: v3_8b CVS $Revision: 1.922 $ OS: Linux 2.6.9-89.0.9.ELsmp i686 Release: Scientific Linux Fermi LTS release 4.4 (Wilson) User: root(0) Group: root(0) Euser: root(0) Egroup: root(0) Library: garbage Storage Group: test File Family: dcache FF Wrapper: cpio_odc FF Width: 1 Current working directory: stkendca22a.fnal.gov:/home/enstore/dcache-log Command line: /fnal/ups/prd/encp/v3_8b/Linux-2-6-dcache/encp --verbose 4 --threaded --ecrc --bypass-filesystem-max-filesize-check --pnfs-mount /pnfs/fs --shortcut --override-path /pnfs/fnal.gov/usr/fermigrid/volatile/fermilab/timur-test/broken-lib/timur-test-0002 --put-cache 0005000000000000005FA5C8 /diskb/write-pool-2/data/0005000000000000005FA5C8 00:00:59 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:00:59 stkendca22a.fnal.gov 022720 root I ENCP Configuration Server: no such name: 'garbage.library_manager' 00:00:59 stkendca22a.fnal.gov 022720 root E ENCP INFILE=/diskb/write-pool-2/data/0005000000000000005FA5F8 OUTFILE=/pnfs/fnal.gov/usr/fermigrid/volatile/fermilab/timur-test/broken-lib/timur-test-0003 FILESIZE=801512 LABEL= LOCATION= DRIVE= DRIVE_SN= TRANSFER_TIME=0.00 SEEK_TIME=0.00 MOUNT_TIME=0.00 QWAIT_TIME=0.00 TIME2NOW=0.85 CRC= STATUS=KEYERROR Configuration Server: no such name: 'garbage.library_manager' 00:00:59 stkendca22a.fnal.gov 022758 root I ENCP Configuration Server: no such name: 'garbage.library_manager' 00:00:59 stkendca22a.fnal.gov 022758 root E ENCP INFILE=/diskb/write-pool-2/data/0005000000000000005FA590 OUTFILE=/pnfs/fnal.gov/usr/fermigrid/volatile/fermilab/timur-test/broken-lib/timur-test-0001 FILESIZE=801512 LABEL= LOCATION= DRIVE= DRIVE_SN= TRANSFER_TIME=0.00 SEEK_TIME=0.00 MOUNT_TIME=0.00 QWAIT_TIME=0.00 TIME2NOW=0.87 CRC= STATUS=KEYERROR Configuration Server: no such name: 'garbage.library_manager' 00:00:59 stkendca22a.fnal.gov 022776 root I ENCP Configuration Server: no such name: 'garbage.library_manager' 00:00:59 stkendca22a.fnal.gov 022776 root E ENCP INFILE=/diskb/write-pool-2/data/0005000000000000005FA5C8 OUTFILE=/pnfs/fnal.gov/usr/fermigrid/volatile/fermilab/timur-test/broken-lib/timur-test-0002 FILESIZE=801512 LABEL= LOCATION= DRIVE= DRIVE_SN= TRANSFER_TIME=0.00 SEEK_TIME=0.00 MOUNT_TIME=0.00 QWAIT_TIME=0.00 TIME2NOW=0.90 CRC= STATUS=KEYERROR Configuration Server: no such name: 'garbage.library_manager' 00:01:00 stkensrv4n.fnal.gov 003689 enstore I SL8500G1MC MC Overflow: 4 mount VOR381 0,5,1,3 00:01:00 stkenmvr149a.fnal.gov 006318 root E LTO4_49MV loadvol ('MEDIA_CHANGER_QUEUE_FULL', 0, 'retry later') Thread media_thread 00:01:00 stkenmvr149a.fnal.gov 006318 root I LTO4_49MV Media Changer returned MEDIA_CHANGER_QUEUE_FULL Thread media_thread 00:01:00 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM 00:01:01 stkenmvr143a.fnal.gov 028425 root I LTO4_43MV tape position: target 89 current 0 1 Thread seek_thread 00:01:01 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:01:01 stkensrv4n.fnal.gov 030269 enstore I SL8500G1MC query vol VOR198 => 0,VOR198 home 0, 4,25, 2, 1 LTO-800G 00:01:01 stkensrv4n.fnal.gov 030274 enstore I SL8500G1MC query drive 0,6,1,11 => 0,0,6,1,11 online in use VOR424 IBM-LTO4 00:01:02 stkenmvr203a.fnal.gov 006116 root I LTO4_103MV written bytes 2107477504/2107477504, blocks 16079 header 292 trailer 335 Thread tape_thread 00:01:02 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM 00:01:02 stkensrv4n.fnal.gov 003689 enstore I SL8500G1MC MC Overflow: 4 mount VOP202 0,5,1,12 00:01:02 stkenmvr151a.fnal.gov 013228 root E LTO4_51MV loadvol ('MEDIA_CHANGER_QUEUE_FULL', 0, 'retry later') Thread media_thread 00:01:02 stkenmvr151a.fnal.gov 013228 root I LTO4_51MV Media Changer returned MEDIA_CHANGER_QUEUE_FULL Thread media_thread 00:01:03 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:01:03 stkensrv4n.fnal.gov 029338 enstore I SL8500G1MC mount VOR202 0,7,1,3 => 0,Mount: VOR202 mounted on 0, 7, 1, 3 00:01:03 stkensrv4n.fnal.gov 003689 enstore I SL8500G1MC FINISHED mount VOR202 0,7,1,3 returned ('ok', 0, 'mount VOR202 0,7,1,3 => 0,Mount: VOR202 mounted on 0, 7, 1, 3', '', '') 00:01:03 last message repeated 1 times 00:01:03 stkensrv0n.fnal.gov 028762 enstore I VOLSRV Updating update volume set sum_mounts = '69', last_access = '2009-11-02 00:01:03' where label = 'VOR202'; 00:01:03 stkenmvr141a.fnal.gov 013963 root I LTO4_41MV MSG_TYPE=MC_LOAD_DONE mounted VOR202 ULTRIUM-TD4 00:01:03 Thread media_thread 00:01:03 stkensrv4n.fnal.gov 030259 enstore I SL8500F1MC dismount VOLUME 1,1,1,8 force => 0,Dismount: Forced dismount of VOO096 from 1, 1, 1, 8 00:01:03 stkensrv4n.fnal.gov 005843 enstore I SL8500F1MC FINISHED dismount VOO096 1,1,1,8 returned ('ok', 0, 'dismount VOLUME 1,1,1,8 force => 0,Dismount: Forced dismount of VOO096 from 1, 1, 1, 8', '', '') 00:01:03 stkenmvr214a.fnal.gov 006260 root I LTO4_114MV dismounted VOO096 ULTRIUM-TD4 00:01:03 Thread media_thread 00:01:03 stkensrv4n.fnal.gov 003689 enstore I SL8500G1MC PROCESSING dismount VON399 0,6,1,4 00:01:03 stkenmvr203a.fnal.gov 006116 root I LTO4_103MV filemarks written. Tape VOR423 absolute location in blocks 914290 Thread tape_thread 00:01:03 stkenmvr203a.fnal.gov 006116 root I LTO4_103MV write_tape timing:pull 0.182191133499 write 10.3795795441 crc_check 9.47263646126 freespace 0.175637245178 block_write 20.210044384 idle 0.303857326508 Thread tape_thread 00:01:03 stkenmvr203a.fnal.gov 006116 root I LTO4_103MV drive stats after write. Tape VOR423 position 37 block 0 block_size 0 bloc_loc 914290 tot_blocks 762939000 BOT 0 read_err 0 write_err 0 bytes 2107477504 block_write_tot 20.210044384 tape_rate 104278717.253 Thread tape_thread 00:01:03 stkenmvr203a.fnal.gov 006116 root I LTO4_103MV remaining bytes info in DB 667462656000 estimated 665355178496 reported from drive 663811072000 position 38 Thread tape_thread 00:01:03 stkenmvr203a.fnal.gov 006116 root I LTO4_103MV new bitfile request {'uid': 0, 'external_label': 'VOR423', 'gid': 0, 'location_cookie': '0000_000000000_0000037', 'complete_crc': 2172903069L, 'sanity_cookie': (65536L, 2361239405L), 'size': 2107476877L} Thread tape_thread 00:01:03 stkenmvr203a.fnal.gov 006116 root I LTO4_103MV set remaining: VOR423 663811072000 0000_000000000_0000038 Thread tape_thread 00:01:03 last message repeated 1 times 00:01:03 stkensrv0n.fnal.gov 028762 enstore I VOLSRV Updating update volume set last_access = '2009-11-02 00:01:03', eod_cookie = '0000_000000000_0000038', non_del_files = '37', remaining_bytes = 663811072000 where label = 'VOR423'; 00:01:03 stkenmvr203a.fnal.gov 006116 root I LTO4_103MV transfer complete volume=VOR423 location=38 Thread tape_thread 00:01:03 last message repeated 1 times 00:01:03 stkensrv0n.fnal.gov 028762 enstore I VOLSRV Updating update volume set sum_wr_access = '37', modification_time = '2009-11-02 00:01:03' where label = 'VOR423'; 00:01:03 stkenmvr203a.fnal.gov 006116 root I LTO4_103MV Sending done to client: {'vc': {'comment': '', 'storage_group': 'cms', 'blocksize': 131072, 'declared': 1255358929.0, 'library': 'CD-LTO4G1', 'si_time': [0.0, 1246283775.0], 'user_inhibit': ['none', 'none'], 'system_inhibit': ['none', 'none'], 'external_label': 'VOR423', 'wrapper': 'cpio_odc', 'remaining_bytes': 667462656000L, 'sum_mounts': 78, 'sum_rd_access': 0, 'file_family_width': 15, 'last_access': 1257141369.0, 'media_type': 'LTO4', 'status': ('ok', None), 'eod_cookie': '0000_000000000_0000037', 'capacity_bytes': 858993459200L, 'non_del_files': 36, 'sum_wr_err': 0, 'sum_wr_access': 36, 'file_family': 'BACKFILL2', 'address': ('131.225.13.59', 7502), 'volume_family': 'cms.BACKFILL2.cpio_odc', 'modification_time': 1257141369.0, 'write_protected': 'n', 'sum_rd_err': 0, 'first_access': 1257122479.0}, 'outfilepath': '/pnfs/fnal.gov/usr/cms/WAX//11/store/backfill/2/data/IansMagicMushroomSoup-T0Test-AnalyzeThisAndGetAFreeBigFatHairyPhD/JetMonitor/RAW/v10_31_pre20replaythingy_v15/000/000/015/68EF89C6-3DC6-DE11-98F1-0030487A3C9A.root', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 25, 'delayed_dismount': None, 'curpri': 25}, 'file_size': 2107476877L, 'mover': {'max_buffer': 2621440000L, 'check_written_file': 600, 'max_failures': 5, 'library': 'CD-LTO4G1.library_manager', 'local_mover': 0, 'single_filemark': 1, 'max_consecutive_failures': 3, 'norestart': 'INQ', 'port': 7863, 'mc_device': '0,6,1,2', 'compression': 0, 'serial_num': '1310050821', 'update_interval_in_bound': 2, 'update_interval': 5, 'syslog_entry': 'sense|st[0-9]', 'status': ('ok', None), 'max_rate': 73400320.0, 'vendor_id': 'IBM', 'driver': 'FTTDriver', 'host': 'stkenmvr203a', 'max_time_in_state': 1200, 'device': 'stkenmvr203a:/dev/rmt/tps0d0n', 'name': 'LTO4_103.mover', 'send_stats': 1, 'product_id': 'ULTRIUM-TD4', 'statistics_path': '/tmp/enstore/enstore/LTO4_103.stat', 'mover_address': ('131.225.13.198', 7863), 'mount_delay': 15, 'check_first_written_file': 0, 'callback_addr': ('131.225.13.198', 57822), 'data_ip': 'stkenmvr203a', 'logname': 'LTO4_103MV', 'hostip': '131.225.13.198', 'do_cleaning': 'No', 'media_changer': 'SL8500G1.media_changer'}, 'ignore_fair_share': None, 'retry': None, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fnal.gov/usr/cms/WAX//11/store/backfill/2/data/IansMagicMushroomSoup-T0Test-AnalyzeThisAndGetAFreeBigFatHairyPhD/JetMonitor/RAW/v10_31_pre20replaythingy_v15/000/000/015/68EF89C6-3DC6-DE11-98F1-0030487A3C9A.root', 'uid': 0, 'uname': 'root', 'mtime': 1257141501, 'rmajor': 0, 'machine': ('Linux', 'cmsstor139.fnal.gov', '2.6.31', '#1 SMP Mon Sep 14 09:43:20 CDT 2009', 'x86_64'), 'sanity_size': 65536, 'type': 'cpio_odc', 'pstat': (16877, 126372832, 23L, 1, 9811, 5063, 512, 1257141485, 1257141485, 1257107163), 'mode_octal': '0100016', 'mode': 33188, 'size_bytes': 2107476877L, 'gname': 'root', 'gid': 0, 'fullname': '/storage/data3/write-pool-3/data/0007000000000000618C8900', 'inode': 126650624L, 'minor': 23}, 'version': 'v3_7d CVS $Revision: 1.922 $ encp', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/storage/data3/write-pool-3/data/0007000000000000618C8900', 'status': ('ok', None), 'outfile': '/pnfs/fnal.gov/usr/cms/WAX//11/store/backfill/2/data/IansMagicMushroomSoup-T0Test-AnalyzeThisAndGetAFreeBigFatHairyPhD/JetMonitor/RAW/v10_31_pre20replaythingy_v15/000/000/015/.(access)(0007000000000000618C8900)', 'fc': {'uid': 0, 'external_label': 'VOR423', 'gid': 0, 'location_cookie': '0000_000000000_0000037', 'complete_crc': 2172903069L, 'bfid': 'CDMS125714166300000', 'sanity_cookie': (65536L, 2361239405L), 'size': 2107476877L}, 'infilepath': '/storage/data3/write-pool-3/data/0007000000000000618C8900', 'at_the_top': 9, 'local_inode': 4572, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.189.110', 59226), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'encp_start_time': 1257141500.8839691, 'seek_time': 125.36377596855164, 'transfer_time': 23.638988971710205, 't0': 1257141500(trunc. 4342) Thread tape_thread 00:01:03 stkenmvr203a.fnal.gov 006116 root I LTO4_103MV Done is sent Thread tape_thread 00:01:03 stkenmvr203a.fnal.gov 006116 root I LTO4_103MV maybe_clean Thread tape_thread 00:01:03 cmsstor139.fnal.gov 005777 root I ENCP Received final dialog for cmsstor139.fnal.gov-1257141501-5777-0. elapsed=163.050sec 00:01:03 cmsstor139.fnal.gov 005777 root I ENCP leaving transfer_file(). 00:01:03 stkenmvr203a.fnal.gov 006116 root I LTO4_103MV returned from maybe_clean Thread tape_thread 00:01:04 cmsstor139.fnal.gov 005777 root I ENCP pf_status = ('ok', None) skip_layer_cleanup = False 00:01:04 stkensrv0n.fnal.gov 028796 enstore I FILSRV Updating update file set deleted = 'n', pnfs_path = '/pnfs/fnal.gov/usr/cms/WAX//11/store/backfill/2/data/IansMagicMushroomSoup-T0Test-AnalyzeThisAndGetAFreeBigFatHairyPhD/JetMonitor/RAW/v10_31_pre20replaythingy_v15/000/000/015/68EF89C6-3DC6-DE11-98F1-0030487A3C9A.root', drive = 'stkenmvr203a:/dev/rmt/tps0d0n:1310050821', pnfs_id = '0007000000000000618C8900' where bfid = 'CDMS125714166300000'; 00:01:04 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM 00:01:04 cmsstor139.fnal.gov 005777 root I ENCP MSG_TYPE=ENCP_XFER write_to_hsm /storage/data3/write-pool-3/data/0007000000000000618C8900 -> /pnfs/fnal.gov/usr/cms/WAX//11/store/backfill/2/data/IansMagicMushroomSoup-T0Test-AnalyzeThisAndGetAFreeBigFatHairyPhD/JetMonitor/RAW/v10_31_pre20replaythingy_v15/000/000/015/68EF89C6-3DC6-DE11-98F1-0030487A3C9A.root: 2107476877 bytes copied to VOR423 at 85 MB/S (97.1 MB/S network) (90.2 MB/S drive) (113 MB/S disk) mover=LTO4_103.mover drive_id=ULTRIUM-TD4 drive_sn=1310050821 drive_vendor=IBM elapsed=163.54 {'storage_group': 'cms', 'encp_crc': 2172903069L, 'disk_rate': 112.8892494897062, 'drive_rate': 90.233631865071175, 'driver': 'FTTDriver', 'transfer_rate': 84.964436681519174, 'mover_interface': 'stkenmvr203a', 'overall_rate': 13.441222990851587, 'media_changer': 'SL8500G1.media_changer', 'network_rate': 84.964436681519174, 'encp_ip': '131.225.189.110', 'unique_id': 'cmsstor139.fnal.gov-1257141501-5777-0'} 00:01:04 cmsstor139.fnal.gov 005777 root I ENCP File /pnfs/fnal.gov/usr/cms/WAX//11/store/backfill/2/data/IansMagicMushroomSoup-T0Test-AnalyzeThisAndGetAFreeBigFatHairyPhD/JetMonitor/RAW/v10_31_pre20replaythingy_v15/000/000/015/68EF89C6-3DC6-DE11-98F1-0030487A3C9A.root copied successfully. 00:01:04 cmsstor139.fnal.gov 005777 root I ENCP Completed transferring 2107476877 bytes in 1 files in 163.54293704 sec. Overall rate = 13.4 MB/sec. Transfer rate = 85 MB/sec. Network rate = 97.1 MB/sec. Drive rate = 90.2 MB/sec. Disk rate = 113 MB/sec. Exit status = 0. 00:01:04 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:01:04 stkensrv4n.fnal.gov 030269 enstore I SL8500G1MC query drive 0,5,1,7 => 0,0,5,1,7 online available IBM-LTO4 00:01:05 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:01:06 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM 00:01:06 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:01:07 last message repeated 1 times 00:01:07 stkenmvr196a.fnal.gov 027637 root I LTO4_96MV Updating stats Thread media_thread 00:01:07 stkenmvr196a.fnal.gov 027637 root I LTO4_96MV writing a tape mark before dismount Thread media_thread 00:01:07 stkensrv4n.fnal.gov 030291 enstore I SL8500G1MC query drive 0,6,1,4 => 0,0,6,1,4 online in use VON399 IBM-LTO4 00:01:08 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM 00:01:08 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:01:08 stkensrv4n.fnal.gov 003689 enstore I SL8500G1MC MC Overflow: 4 mount VOR382 0,6,1,13 00:01:08 stkenmvr204a.fnal.gov 006261 root E LTO4_104MV loadvol ('MEDIA_CHANGER_QUEUE_FULL', 0, 'retry later') Thread media_thread 00:01:08 stkenmvr204a.fnal.gov 006261 root I LTO4_104MV Media Changer returned MEDIA_CHANGER_QUEUE_FULL Thread media_thread 00:01:08 stkensrv4n.fnal.gov 003689 enstore I SL8500G1MC MC Overflow: 4 mount VON584 0,5,1,15 00:01:08 stkenmvr148a.fnal.gov 006010 root E LTO4_48MV loadvol ('MEDIA_CHANGER_QUEUE_FULL', 0, 'retry later') Thread media_thread 00:01:08 stkenmvr148a.fnal.gov 006010 root I LTO4_48MV Media Changer returned MEDIA_CHANGER_QUEUE_FULL Thread media_thread 00:01:10 stkensrv4n.fnal.gov 003689 enstore I SL8500G1MC MC Overflow: 4 mount VOR381 0,5,1,3 00:01:10 stkenmvr149a.fnal.gov 006318 root E LTO4_49MV loadvol ('MEDIA_CHANGER_QUEUE_FULL', 0, 'retry later') Thread media_thread 00:01:10 stkenmvr149a.fnal.gov 006318 root I LTO4_49MV Media Changer returned MEDIA_CHANGER_QUEUE_FULL Thread media_thread 00:01:10 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM 00:01:10 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:01:10 stkenmvr196a.fnal.gov 027637 root I LTO4_96MV Ejecting tape Thread media_thread 00:01:12 stkenmvr144a.fnal.gov 006329 root I LTO4_44MV skip_fm 39 Thread seek_thread 00:01:12 stkenmvr144a.fnal.gov 006329 root I LTO4_44MV write_tape starting, bytes_to_write=2096337408 Thread tape_thread 00:01:12 stkenmvr144a.fnal.gov 006329 root I LTO4_44MV Write starting. Tape VOR426 absolute location in blocks 893515 Thread tape_thread 00:01:12 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM 00:01:12 stkenmvr217a.fnal.gov 007075 root I LTO4_117MV Updating stats Thread media_thread 00:01:12 stkenmvr217a.fnal.gov 007075 root I LTO4_117MV writing a tape mark before dismount Thread media_thread 00:01:12 cmsstor63.fnal.gov 005003 root I ENCP Version: v3_7d CVS $Revision: 1.922 $ encp OS: Linux 2.6.31 x86_64 Release: Rocks release 5.1 (V.I) Scientific Linux SL release 5.2 (Boron) User: root(0) Group: root(0) Euser: root(0) Egroup: root(0) Library: CD-LTO4G1 Storage Group: cms File Family: BACKFILL2 FF Wrapper: cpio_odc FF Width: 15 Current working directory: cmsstor63.fnal.gov:/ Command line: /home/enstore/enstore/bin/encp --verbose 4 --threaded --ecrc --resubmit-timeout 1800 --age-time 30 --delpri 1 --priority 25 --pnfs-mount /pnfs/fs --shortcut --override-path /pnfs/fnal.gov/usr/cms/WAX//11/store/backfill/2/data/IansMagicMushroomSoup-T0Test-AnalyzeThisAndGetAFreeBigFatHairyPhD/JetMonitor/RAW/v10_31_pre20replaythingy_v15/000/000/015/8A623D3F-40C6-DE11-B943-001D09F29533.root --put-cache 0007000000000000618C8FE8 /storage/data2/write-pool-2/data/0007000000000000618C8FE8 00:01:12 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:01:12 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:01:12 cmsstor63.fnal.gov 005003 root I ENCP Creating zero length output files. 00:01:12 cmsstor63.fnal.gov 005003 root I ENCP Sending /pnfs/fnal.gov/usr/cms/WAX//11/store/backfill/2/data/IansMagicMushroomSoup-T0Test-AnalyzeThisAndGetAFreeBigFatHairyPhD/JetMonitor/RAW/v10_31_pre20replaythingy_v15/000/000/015/8A623D3F-40C6-DE11-B943-001D09F29533.root write request to LM: unique_id: cmsstor63.fnal.gov-1257141672-5003-0 inputfile: /storage/data2/write-pool-2/data/0007000000000000618C8FE8 outputfile: /pnfs/fnal.gov/usr/cms/WAX//11/store/backfill/2/data/IansMagicMushroomSoup-T0Test-AnalyzeThisAndGetAFreeBigFatHairyPhD/JetMonitor/RAW/v10_31_pre20replaythingy_v15/000/000/015/.(access)(0007000000000000618C8FE8) 00:01:12 stkensrv4n.fnal.gov 003689 enstore I SL8500G1MC MC Overflow: 4 mount VOP202 0,5,1,12 00:01:12 stkenmvr151a.fnal.gov 013228 root E LTO4_51MV loadvol ('MEDIA_CHANGER_QUEUE_FULL', 0, 'retry later') Thread media_thread 00:01:12 stkenmvr151a.fnal.gov 013228 root I LTO4_51MV Media Changer returned MEDIA_CHANGER_QUEUE_FULL Thread media_thread 00:01:12 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM MSG_TYPE=ADD_TO_LMQ request added to pending queue for storage group : cms 00:01:12 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM write Q'd /storage/data2/write-pool-2/data/0007000000000000618C8FE8 (cmsstor63.fnal.gov-1257141672-5003-0) -> /pnfs/fnal.gov/usr/cms/WAX//11/store/backfill/2/data/IansMagicMushroomSoup-T0Test-AnalyzeThisAndGetAFreeBigFatHairyPhD/JetMonitor/RAW/v10_31_pre20replaythingy_v15/000/000/015/8A623D3F-40C6-DE11-B943-001D09F29533.root : library=CD-LTO4G1 family=BACKFILL2 requester:root volume_family:cms.BACKFILL2.cpio_odc 00:01:13 cmsstor63.fnal.gov 005003 root I ENCP Listening for control socket at: ('131.225.204.239', 47703) 00:01:13 cmsstor63.fnal.gov 005003 root I ENCP Waiting for mover to connect control socket at address: ('131.225.204.239', 47703) elapsed=1.180sec 00:01:13 stkenmvr146a.fnal.gov 005990 root I LTO4_46MV selective CRC check after writing file completed successfuly Thread tape_thread 00:01:13 stkenmvr146a.fnal.gov 005990 root I LTO4_46MV tape position: target 199 current 198 26226 Thread tape_thread 00:01:13 stkenmvr146a.fnal.gov 005990 root I LTO4_46MV skip_fm 1 Thread tape_thread 00:01:13 stkenmvr146a.fnal.gov 005990 root I LTO4_46MV drive stats after crc check. Tape VOR200 block 0 block_size 0 bloc_loc 4724302 tot_blocks 762939000 BOT 0 read_err 0 write_err 0 bytes 3437402848 block_read_tot 29.1203844547 tape_rate 118041121.79 Thread tape_thread 00:01:13 stkenmvr146a.fnal.gov 005990 root I LTO4_46MV remaining bytes info in DB 201606144000 estimated 198168740864 reported from drive 198424576000 position 199 Thread tape_thread 00:01:13 stkenmvr146a.fnal.gov 005990 root I LTO4_46MV new bitfile request {'uid': 0, 'external_label': 'VOR200', 'gid': 0, 'location_cookie': '0000_000000000_0000198', 'complete_crc': 1174906552L, 'sanity_cookie': (65536L, 750575638L), 'size': 3437402562L} Thread tape_thread 00:01:13 stkenmvr146a.fnal.gov 005990 root I LTO4_46MV set remaining: VOR200 198424576000 0000_000000000_0000199 Thread tape_thread 00:01:13 last message repeated 1 times 00:01:13 stkensrv0n.fnal.gov 028762 enstore I VOLSRV Updating update volume set last_access = '2009-11-02 00:01:13', eod_cookie = '0000_000000000_0000199', non_del_files = '198', remaining_bytes = 198424576000 where label = 'VOR200'; 00:01:13 stkenmvr146a.fnal.gov 005990 root I LTO4_46MV transfer complete volume=VOR200 location=199 Thread tape_thread 00:01:13 last message repeated 1 times 00:01:13 stkensrv0n.fnal.gov 028762 enstore I VOLSRV Updating update volume set sum_wr_access = '198', modification_time = '2009-11-02 00:01:13' where label = 'VOR200'; 00:01:13 stkenmvr146a.fnal.gov 005990 root I LTO4_46MV Sending done to client: {'vc': {'comment': '', 'storage_group': 'cms', 'blocksize': 131072, 'declared': 1255358745.0, 'library': 'CD-LTO4G1', 'si_time': [0.0, 1238581927.0], 'user_inhibit': ['none', 'none'], 'system_inhibit': ['none', 'none'], 'external_label': 'VOR200', 'wrapper': 'cpio_odc', 'remaining_bytes': 201606144000L, 'sum_mounts': 91, 'sum_rd_access': 0, 'file_family_width': 15, 'last_access': 1257141582.0, 'media_type': 'LTO4', 'status': ('ok', None), 'eod_cookie': '0000_000000000_0000198', 'capacity_bytes': 858993459200L, 'non_del_files': 197, 'sum_wr_err': 0, 'sum_wr_access': 197, 'file_family': 'BACKFILL2', 'address': ('131.225.13.59', 7502), 'volume_family': 'cms.BACKFILL2.cpio_odc', 'modification_time': 1257141582.0, 'write_protected': 'n', 'sum_rd_err': 0, 'first_access': 1257103365.0}, 'outfilepath': '/pnfs/fnal.gov/usr/cms/WAX//11/store/backfill/2/data/IansMagicMushroomSoup-T0Test-AnalyzeThisAndGetAFreeBigFatHairyPhD/Mu_X/RAW/v10_31_pre20replaythingy_v15/000/000/016/CC6314C9-5AC6-DE11-B573-001617C3B66C.root', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 25, 'delayed_dismount': None, 'curpri': 25}, 'file_size': 3437402562L, 'mover': {'max_buffer': 2621440000L, 'check_written_file': 600, 'max_failures': 5, 'library': 'CD-LTO4G1.library_manager', 'local_mover': 0, 'dismount_delay': 30, 'max_consecutive_failures': 3, 'norestart': 'INQ', 'port': 7846, 'mc_device': '0,6,1,12', 'compression': 0, 'serial_num': '9311032672', 'update_interval_in_bound': 2, 'update_interval': 5, 'syslog_entry': 'sense|st[0-9]', 'status': ('ok', None), 'max_rate': 73400320.0, 'vendor_id': 'IBM', 'driver': 'FTTDriver', 'host': 'stkenmvr146a', 'max_time_in_state': 1200, 'device': 'stkenmvr146a:/dev/rmt/tps0d0n', 'name': 'LTO4_46.mover', 'send_stats': 1, 'product_id': 'ULTRIUM-TD4', 'statistics_path': '/tmp/enstore/enstore/LTO4_46.stat', 'single_filemark': 1, 'mount_delay': 15, 'mover_address': ('131.225.13.174', 7846), 'check_first_written_file': 0, 'callback_addr': ('131.225.13.174', 56673), 'data_ip': 'stkenmvr146a', 'logname': 'LTO4_46MV', 'hostip': '131.225.13.174', 'do_cleaning': 'No', 'media_changer': 'SL8500G1.media_changer', 'min_buffer': 67108864L}, 'ignore_fair_share': None, 'retry': None, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fnal.gov/usr/cms/WAX//11/store/backfill/2/data/IansMagicMushroomSoup-T0Test-AnalyzeThisAndGetAFreeBigFatHairyPhD/Mu_X/RAW/v10_31_pre20replaythingy_v15/000/000/016/CC6314C9-5AC6-DE11-B573-001617C3B66C.root', 'uid': 0, 'uname': 'root', 'mtime': 1257141582, 'rmajor': 0, 'machine': ('Linux', 'cmsstor188.fnal.gov', '2.6.31', '#1 SMP Mon Sep 14 09:43:20 CDT 2009', 'x86_64'), 'sanity_size': 65536, 'type': 'cpio_odc', 'pstat': (16877, 126646856, 23L, 1, 9811, 5063, 512, 1257141149, 1257141149, 1257140950), 'mode_octal': '0100016', 'mode': 33188, 'size_bytes': 3437402562L, 'gname': 'root', 'gid': 0, 'fullname': '/storage/data3/write-pool-3/data/0007000000000000618C8660', 'inode': 126649952L, 'minor': 23}, 'version': 'v3_7d CVS $Revision: 1.922 $ encp', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/storage/data3/write-pool-3/data/0007000000000000618C8660', 'status': ('ok', None), 'outfile': '/pnfs/fnal.gov/usr/cms/WAX//11/store/backfill/2/data/IansMagicMushroomSoup-T0Test-AnalyzeThisAndGetAFreeBigFatHairyPhD/Mu_X/RAW/v10_31_pre20replaythingy_v15/000/000/016/.(access)(0007000000000000618C8660)', 'fc': {'uid': 0, 'external_label': 'VOR200', 'gid': 0, 'location_cookie': '0000_000000000_0000198', 'complete_crc': 1174906552L, 'bfid': 'CDMS125714167300000', 'sanity_cookie': (65536L, 750575638L), 'size': 3437402562L}, 'infilepath': '/storage/data3/write-pool-3/data/0007000000000000618C8660', 'at_the_top': 1, 'local_inode': 3627, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.207.11', 54656), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'encp_start_time': 1257141582.3097839, 'seek_time': 0.035001993179321289, 'transfer_time': 89.8949(trunc. 4408) Thread tape_thread 00:01:13 stkenmvr146a.fnal.gov 005990 root I LTO4_46MV Done is sent Thread tape_thread 00:01:13 stkenmvr146a.fnal.gov 005990 root I LTO4_46MV maybe_clean Thread tape_thread 00:01:13 cmsstor188.fnal.gov 019677 root I ENCP Received final dialog for cmsstor188.fnal.gov-1257141582-19677-0. elapsed=91.110sec 00:01:13 cmsstor188.fnal.gov 019677 root I ENCP leaving transfer_file(). 00:01:13 stkenmvr146a.fnal.gov 005990 root I LTO4_46MV returned from maybe_clean Thread tape_thread 00:01:13 stkenmvr210a.fnal.gov 011436 root I LTO4_110MV tape position: target 107 current 0 1 Thread seek_thread 00:01:13 cmsstor188.fnal.gov 019677 root I ENCP pf_status = ('ok', None) skip_layer_cleanup = False 00:01:14 stkensrv0n.fnal.gov 028796 enstore I FILSRV Updating update file set deleted = 'n', pnfs_path = '/pnfs/fnal.gov/usr/cms/WAX//11/store/backfill/2/data/IansMagicMushroomSoup-T0Test-AnalyzeThisAndGetAFreeBigFatHairyPhD/Mu_X/RAW/v10_31_pre20replaythingy_v15/000/000/016/CC6314C9-5AC6-DE11-B573-001617C3B66C.root', drive = 'stkenmvr146a:/dev/rmt/tps0d0n:9311032672', pnfs_id = '0007000000000000618C8660' where bfid = 'CDMS125714167300000'; 00:01:14 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:01:14 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:01:14 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM write_to_hsm next work on vol=VOR200 mover=LTO4_46.mover requester:root 00:01:14 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM HAVE_BOUND:sending write_to_hsm /pnfs/fnal.gov/usr/cms/WAX//11/store/backfill/2/data/IansMagicMushroomSoup-T0Test-AnalyzeThisAndGetAFreeBigFatHairyPhD/JetMonitor/RAW/v10_31_pre20replaythingy_v15/000/000/015/8A623D3F-40C6-DE11-B943-001D09F29533.root to mover LTO4_46.mover ('131.225.13.174', 7846) DEL_DISM None 00:01:14 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM MSG_TYPE=ADD_TO_LMQ request added to work at movers queue for storage group : cms 00:01:14 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM HAVE_BOUND:sending {'vc': {'storage_group': 'cms', 'external_label': 'VOR200', 'library': 'CD-LTO4G1', 'file_family': 'BACKFILL2', 'wrapper': 'cpio_odc', 'address': ('131.225.13.59', 7502), 'volume_family': 'cms.BACKFILL2.cpio_odc', 'file_family_width': 15}, 'outfilepath': '/pnfs/fnal.gov/usr/cms/WAX//11/store/backfill/2/data/IansMagicMushroomSoup-T0Test-AnalyzeThisAndGetAFreeBigFatHairyPhD/JetMonitor/RAW/v10_31_pre20replaythingy_v15/000/000/015/8A623D3F-40C6-DE11-B943-001D09F29533.root', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 25, 'delayed_dismount': None, 'curpri': 25}, 'file_size': 2098472578L, 'mover': 'LTO4_46.mover', 'ignore_fair_share': None, 'retry': None, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fnal.gov/usr/cms/WAX//11/store/backfill/2/data/IansMagicMushroomSoup-T0Test-AnalyzeThisAndGetAFreeBigFatHairyPhD/JetMonitor/RAW/v10_31_pre20replaythingy_v15/000/000/015/8A623D3F-40C6-DE11-B943-001D09F29533.root', 'uid': 0, 'uname': 'root', 'type': 'cpio_odc', 'gid': 0, 'mtime': 1257141672, 'rmajor': 0, 'machine': ('Linux', 'cmsstor63.fnal.gov', '2.6.31', '#1 SMP Mon Sep 14 09:43:20 CDT 2009', 'x86_64'), 'sanity_size': 65536, 'pstat': (16877, 126372832, 21L, 1, 9811, 5063, 512, 1257141604, 1257141604, 1257107163), 'mode_octal': '0100016', 'mode': 33188, 'gname': 'root', 'size_bytes': 2098472578L, 'fullname': '/storage/data2/write-pool-2/data/0007000000000000618C8FE8', 'inode': 126652392L, 'minor': 21}, 'version': 'v3_7d CVS $Revision: 1.922 $ encp', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/storage/data2/write-pool-2/data/0007000000000000618C8FE8', 'status': ('ok', None), 'outfile': '/pnfs/fnal.gov/usr/cms/WAX//11/store/backfill/2/data/IansMagicMushroomSoup-T0Test-AnalyzeThisAndGetAFreeBigFatHairyPhD/JetMonitor/RAW/v10_31_pre20replaythingy_v15/000/000/015/.(access)(0007000000000000618C8FE8)', 'fc': {'size': 2098472578L, 'external_label': 'VOR200', 'address': ('131.225.13.59', 7501)}, 'infilepath': '/storage/data2/write-pool-2/data/0007000000000000618C8FE8', 'at_the_top': 9, 'local_inode': 1073742074, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.204.239', 47703), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'lm_dequeued': 1257141674.442162, 'encp_start_time': 1257141671.8277791, 'job_queued': 1257141672.9413431, 't0': 1257141671, 'in_queue': 1.5007588863372803}, 'user_level': 'admin', 'unique_id': 'cmsstor63.fnal.gov-1257141672-5003-0'} to mover 00:01:14 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:01:14 stkenmvr146a.fnal.gov 005990 root I LTO4_46MV WRITE_TO_HSM Thread MainThread 00:01:14 stkenmvr146a.fnal.gov 005990 root I LTO4_46MV SENDING {'vc': {'storage_group': 'cms', 'external_label': 'VOR200', 'library': 'CD-LTO4G1', 'file_family': 'BACKFILL2', 'wrapper': 'cpio_odc', 'address': ('131.225.13.59', 7502), 'volume_family': 'cms.BACKFILL2.cpio_odc', 'file_family_width': 15}, 'outfilepath': '/pnfs/fnal.gov/usr/cms/WAX//11/store/backfill/2/data/IansMagicMushroomSoup-T0Test-AnalyzeThisAndGetAFreeBigFatHairyPhD/JetMonitor/RAW/v10_31_pre20replaythingy_v15/000/000/015/8A623D3F-40C6-DE11-B943-001D09F29533.root', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 25, 'delayed_dismount': None, 'curpri': 25}, 'file_size': 2098472578L, 'mover': {'max_buffer': 2621440000L, 'check_written_file': 600, 'max_failures': 5, 'library': 'CD-LTO4G1.library_manager', 'local_mover': 0, 'dismount_delay': 30, 'max_consecutive_failures': 3, 'norestart': 'INQ', 'port': 7846, 'mc_device': '0,6,1,12', 'compression': 0, 'serial_num': '9311032672', 'update_interval_in_bound': 2, 'update_interval': 5, 'syslog_entry': 'sense|st[0-9]', 'status': ('ok', None), 'max_rate': 73400320.0, 'vendor_id': 'IBM', 'driver': 'FTTDriver', 'host': 'stkenmvr146a', 'max_time_in_state': 1200, 'device': 'stkenmvr146a:/dev/rmt/tps0d0n', 'name': 'LTO4_46.mover', 'send_stats': 1, 'product_id': 'ULTRIUM-TD4', 'statistics_path': '/tmp/enstore/enstore/LTO4_46.stat', 'single_filemark': 1, 'mount_delay': 15, 'mover_address': ('131.225.13.174', 7846), 'check_first_written_file': 0, 'callback_addr': ('131.225.13.174', 56681), 'data_ip': 'stkenmvr146a', 'logname': 'LTO4_46MV', 'hostip': '131.225.13.174', 'do_cleaning': 'No', 'media_changer': 'SL8500G1.media_changer', 'min_buffer': 67108864L}, 'ignore_fair_share': None, 'retry': None, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fnal.gov/usr/cms/WAX//11/store/backfill/2/data/IansMagicMushroomSoup-T0Test-AnalyzeThisAndGetAFreeBigFatHairyPhD/JetMonitor/RAW/v10_31_pre20replaythingy_v15/000/000/015/8A623D3F-40C6-DE11-B943-001D09F29533.root', 'uid': 0, 'uname': 'root', 'mtime': 1257141672, 'rmajor': 0, 'machine': ('Linux', 'cmsstor63.fnal.gov', '2.6.31', '#1 SMP Mon Sep 14 09:43:20 CDT 2009', 'x86_64'), 'sanity_size': 65536, 'type': 'cpio_odc', 'pstat': (16877, 126372832, 21L, 1, 9811, 5063, 512, 1257141604, 1257141604, 1257107163), 'mode_octal': '0100016', 'mode': 33188, 'size_bytes': 2098472578L, 'gname': 'root', 'gid': 0, 'fullname': '/storage/data2/write-pool-2/data/0007000000000000618C8FE8', 'inode': 126652392L, 'minor': 21}, 'version': 'v3_7d CVS $Revision: 1.922 $ encp', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/storage/data2/write-pool-2/data/0007000000000000618C8FE8', 'status': ('ok', None), 'outfile': '/pnfs/fnal.gov/usr/cms/WAX//11/store/backfill/2/data/IansMagicMushroomSoup-T0Test-AnalyzeThisAndGetAFreeBigFatHairyPhD/JetMonitor/RAW/v10_31_pre20replaythingy_v15/000/000/015/.(access)(0007000000000000618C8FE8)', 'fc': {'size': 2098472578L, 'external_label': 'VOR200', 'address': ('131.225.13.59', 7501)}, 'infilepath': '/storage/data2/write-pool-2/data/0007000000000000618C8FE8', 'at_the_top': 9, 'local_inode': 1073742074, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.204.239', 47703), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'lm_dequeued': 1257141674.442162, 'encp_start_time': 1257141671.8277791, 'job_queued': 1257141672.9413431, 't0': 1257141671, 'in_queue': 1.5007588863372803}, 'user_level': 'admin', 'unique_id': 'cmsstor63.fnal.gov-1257141672-5003-0'} Thread client_connect_thread 00:01:14 cmsstor63.fnal.gov 005003 root I ENCP Control socket ('131.225.204.239', 47703) is connected to ('131.225.13.174', 56682) for cmsstor63.fnal.gov-1257141672-5003-0. elapsed=2.690sec 00:01:14 cmsstor63.fnal.gov 005003 root I ENCP Received callback ticket from mover LTO4_46.mover for transfer cmsstor63.fnal.gov-1257141672-5003-0. elapsed=2.700sec 00:01:14 cmsstor63.fnal.gov 005003 root I ENCP Atempting to connect data socket to mover at ('131.225.13.174', 56681). 00:01:14 cmsstor63.fnal.gov 005003 root I ENCP Connecting data socket to mover (('131.225.13.174', 56681)) with interface 131.225.204.239. elapsed=2.700sec 00:01:14 cmsstor63.fnal.gov 005003 root I ENCP Data socket ('131.225.204.239', 47286) is connected to ('131.225.13.174', 56681). elapsed=2.700sec 00:01:14 cmsstor63.fnal.gov 005003 root I ENCP Opening local file. 00:01:14 stkenmvr146a.fnal.gov 005990 root I LTO4_46MV tape position: target 199 current 199 0 Thread seek_thread 00:01:14 stkenmvr146a.fnal.gov 005990 root I LTO4_46MV write_tape starting, bytes_to_write=2098472960 Thread tape_thread 00:01:14 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:01:14 stkenmvr146a.fnal.gov 005990 root I LTO4_46MV Write starting. Tape VOR200 absolute location in blocks 4750529 Thread tape_thread 00:01:14 cmsstor63.fnal.gov 005003 root I ENCP pf_status = ('ok', None) skip_layer_cleanup = False 00:01:15 cmsstor188.fnal.gov 019677 root I ENCP MSG_TYPE=ENCP_XFER write_to_hsm /storage/data3/write-pool-3/data/0007000000000000618C8660 -> /pnfs/fnal.gov/usr/cms/WAX//11/store/backfill/2/data/IansMagicMushroomSoup-T0Test-AnalyzeThisAndGetAFreeBigFatHairyPhD/Mu_X/RAW/v10_31_pre20replaythingy_v15/000/000/016/CC6314C9-5AC6-DE11-B573-001617C3B66C.root: 3437402562 bytes copied to VOR200 at 36.5 MB/S (97.3 MB/S network) (115 MB/S drive) (113 MB/S disk) mover=LTO4_46.mover drive_id=ULTRIUM-TD4 drive_sn=9311032672 drive_vendor=IBM elapsed=92.777 {'storage_group': 'cms', 'encp_crc': 1174906552L, 'disk_rate': 112.9935458475926, 'drive_rate': 114.6592461435375, 'driver': 'FTTDriver', 'transfer_rate': 36.460667526026093, 'mover_interface': 'stkenmvr146a', 'overall_rate': 35.759906332709647, 'media_changer': 'SL8500G1.media_changer', 'network_rate': 36.460667526026093, 'encp_ip': '131.225.207.11', 'unique_id': 'cmsstor188.fnal.gov-1257141582-19677-0'} 00:01:15 cmsstor188.fnal.gov 019677 root I ENCP File /pnfs/fnal.gov/usr/cms/WAX//11/store/backfill/2/data/IansMagicMushroomSoup-T0Test-AnalyzeThisAndGetAFreeBigFatHairyPhD/Mu_X/RAW/v10_31_pre20replaythingy_v15/000/000/016/CC6314C9-5AC6-DE11-B573-001617C3B66C.root copied successfully. 00:01:15 cmsstor188.fnal.gov 019677 root I ENCP Completed transferring 3437402562 bytes in 1 files in 92.7778100967 sec. Overall rate = 35.8 MB/sec. Transfer rate = 36.5 MB/sec. Network rate = 97.3 MB/sec. Drive rate = 115 MB/sec. Disk rate = 113 MB/sec. Exit status = 0. 00:01:15 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:01:15 cmsstor194.fnal.gov 016956 root I ENCP Version: v3_7d CVS $Revision: 1.922 $ encp OS: Linux 2.6.31 x86_64 Release: Scientific Linux SL release 5.2 (Boron) Rocks release 5.1 (V.I) User: root(0) Group: root(0) Euser: root(0) Egroup: root(0) Library: CD-LTO4G1 Storage Group: cms File Family: BACKFILL2 FF Wrapper: cpio_odc FF Width: 15 Current working directory: cmsstor194.fnal.gov:/ Command line: /home/enstore/enstore/bin/encp --verbose 4 --threaded --ecrc --resubmit-timeout 1800 --age-time 30 --delpri 1 --priority 25 --pnfs-mount /pnfs/fs --shortcut --override-path /pnfs/fnal.gov/usr/cms/WAX//11/store/backfill/2/data/IansMagicMushroomSoup-T0Test-AnalyzeThisAndGetAFreeBigFatHairyPhD/JetMonitor/RAW/v10_31_pre20replaythingy_v15/000/000/015/80F0E71A-45C6-DE11-A8B3-001D09F29597.root --put-cache 0007000000000000618C8FA8 /storage/data1/write-pool-1/data/0007000000000000618C8FA8 00:01:16 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:01:17 cmsstor194.fnal.gov 016956 root I ENCP Creating zero length output files. 00:01:17 cmsstor194.fnal.gov 016956 root I ENCP Sending /pnfs/fnal.gov/usr/cms/WAX//11/store/backfill/2/data/IansMagicMushroomSoup-T0Test-AnalyzeThisAndGetAFreeBigFatHairyPhD/JetMonitor/RAW/v10_31_pre20replaythingy_v15/000/000/015/80F0E71A-45C6-DE11-A8B3-001D09F29597.root write request to LM: unique_id: cmsstor194.fnal.gov-1257141677-16956-0 inputfile: /storage/data1/write-pool-1/data/0007000000000000618C8FA8 outputfile: /pnfs/fnal.gov/usr/cms/WAX//11/store/backfill/2/data/IansMagicMushroomSoup-T0Test-AnalyzeThisAndGetAFreeBigFatHairyPhD/JetMonitor/RAW/v10_31_pre20replaythingy_v15/000/000/015/.(access)(0007000000000000618C8FA8) 00:01:17 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM MSG_TYPE=ADD_TO_LMQ request added to pending queue for storage group : cms 00:01:17 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM write Q'd /storage/data1/write-pool-1/data/0007000000000000618C8FA8 (cmsstor194.fnal.gov-1257141677-16956-0) -> /pnfs/fnal.gov/usr/cms/WAX//11/store/backfill/2/data/IansMagicMushroomSoup-T0Test-AnalyzeThisAndGetAFreeBigFatHairyPhD/JetMonitor/RAW/v10_31_pre20replaythingy_v15/000/000/015/80F0E71A-45C6-DE11-A8B3-001D09F29597.root : library=CD-LTO4G1 family=BACKFILL2 requester:root volume_family:cms.BACKFILL2.cpio_odc 00:01:17 cmsstor194.fnal.gov 016956 root I ENCP Listening for control socket at: ('131.225.207.19', 41677) 00:01:17 cmsstor194.fnal.gov 016956 root I ENCP Waiting for mover to connect control socket at address: ('131.225.207.19', 41677) elapsed=2.070sec 00:01:17 stkenmvr217a.fnal.gov 007075 root I LTO4_117MV Ejecting tape Thread media_thread 00:01:18 cmsstor197.fnal.gov 008600 root I ENCP Version: v3_7d CVS $Revision: 1.922 $ encp OS: Linux 2.6.31 x86_64 Release: Scientific Linux SL release 5.2 (Boron) Rocks release 5.1 (V.I) User: root(0) Group: root(0) Euser: root(0) Egroup: root(0) Library: CD-LTO4G1 Storage Group: cms File Family: BACKFILL2 FF Wrapper: cpio_odc FF Width: 15 Current working directory: cmsstor197.fnal.gov:/root Command line: /home/enstore/enstore/bin/encp --verbose 4 --threaded --ecrc --resubmit-timeout 1800 --age-time 30 --delpri 1 --priority 25 --pnfs-mount /pnfs/fs --shortcut --override-path /pnfs/fnal.gov/usr/cms/WAX//11/store/backfill/2/data/IansMagicMushroomSoup-T0Test-AnalyzeThisAndGetAFreeBigFatHairyPhD/JetMonitor/RAW/v10_31_pre20replaythingy_v15/000/000/015/6270F549-3DC6-DE11-AC53-000423D99A8E.root --put-cache 0007000000000000618C8FF0 /storage/data1/write-pool-1/data/0007000000000000618C8FF0 00:01:18 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:01:18 stkensrv4n.fnal.gov 003689 enstore I SL8500G1MC MC Overflow: 4 mount VOR382 0,6,1,13 00:01:18 stkenmvr204a.fnal.gov 006261 root E LTO4_104MV loadvol ('MEDIA_CHANGER_QUEUE_FULL', 0, 'retry later') Thread media_thread 00:01:18 stkenmvr204a.fnal.gov 006261 root I LTO4_104MV Media Changer returned MEDIA_CHANGER_QUEUE_FULL Thread media_thread 00:01:18 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM write_to_hsm next work on vol=VOR423 mover=LTO4_103.mover requester:root 00:01:18 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM HAVE_BOUND:sending write_to_hsm /pnfs/fnal.gov/usr/cms/WAX//11/store/backfill/2/data/IansMagicMushroomSoup-T0Test-AnalyzeThisAndGetAFreeBigFatHairyPhD/JetMonitor/RAW/v10_31_pre20replaythingy_v15/000/000/015/80F0E71A-45C6-DE11-A8B3-001D09F29597.root to mover LTO4_103.mover ('131.225.13.198', 7863) DEL_DISM None 00:01:18 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM MSG_TYPE=ADD_TO_LMQ request added to work at movers queue for storage group : cms 00:01:18 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM HAVE_BOUND:sending {'vc': {'storage_group': 'cms', 'external_label': 'VOR423', 'library': 'CD-LTO4G1', 'file_family': 'BACKFILL2', 'wrapper': 'cpio_odc', 'address': ('131.225.13.59', 7502), 'volume_family': 'cms.BACKFILL2.cpio_odc', 'file_family_width': 15}, 'outfilepath': '/pnfs/fnal.gov/usr/cms/WAX//11/store/backfill/2/data/IansMagicMushroomSoup-T0Test-AnalyzeThisAndGetAFreeBigFatHairyPhD/JetMonitor/RAW/v10_31_pre20replaythingy_v15/000/000/015/80F0E71A-45C6-DE11-A8B3-001D09F29597.root', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 25, 'delayed_dismount': None, 'curpri': 25}, 'file_size': 2099839175L, 'mover': 'LTO4_103.mover', 'ignore_fair_share': None, 'retry': None, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fnal.gov/usr/cms/WAX//11/store/backfill/2/data/IansMagicMushroomSoup-T0Test-AnalyzeThisAndGetAFreeBigFatHairyPhD/JetMonitor/RAW/v10_31_pre20replaythingy_v15/000/000/015/80F0E71A-45C6-DE11-A8B3-001D09F29597.root', 'uid': 0, 'uname': 'root', 'type': 'cpio_odc', 'gid': 0, 'mtime': 1257141677, 'rmajor': 0, 'machine': ('Linux', 'cmsstor194.fnal.gov', '2.6.31', '#1 SMP Mon Sep 14 09:43:20 CDT 2009', 'x86_64'), 'sanity_size': 65536, 'pstat': (16877, 126372832, 21L, 1, 9811, 5063, 512, 1257141604, 1257141604, 1257107163), 'mode_octal': '0100016', 'mode': 33188, 'gname': 'root', 'size_bytes': 2099839175L, 'fullname': '/storage/data1/write-pool-1/data/0007000000000000618C8FA8', 'inode': 126652328L, 'minor': 21}, 'version': 'v3_7d CVS $Revision: 1.922 $ encp', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/storage/data1/write-pool-1/data/0007000000000000618C8FA8', 'status': ('ok', None), 'outfile': '/pnfs/fnal.gov/usr/cms/WAX//11/store/backfill/2/data/IansMagicMushroomSoup-T0Test-AnalyzeThisAndGetAFreeBigFatHairyPhD/JetMonitor/RAW/v10_31_pre20replaythingy_v15/000/000/015/.(access)(0007000000000000618C8FA8)', 'fc': {'size': 2099839175L, 'external_label': 'VOR423', 'address': ('131.225.13.59', 7501)}, 'infilepath': '/storage/data1/write-pool-1/data/0007000000000000618C8FA8', 'at_the_top': 5, 'local_inode': 7339, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.207.19', 41677), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'lm_dequeued': 1257141678.6338661, 'encp_start_time': 1257141675.1245601, 'job_queued': 1257141677.1714289, 't0': 1257141675, 'in_queue': 1.4623761177062988}, 'user_level': 'admin', 'unique_id': 'cmsstor194.fnal.gov-1257141677-16956-0'} to mover 00:01:18 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:01:18 stkenmvr203a.fnal.gov 006116 root I LTO4_103MV WRITE_TO_HSM Thread MainThread 00:01:18 stkenmvr203a.fnal.gov 006116 root I LTO4_103MV SENDING {'vc': {'storage_group': 'cms', 'external_label': 'VOR423', 'library': 'CD-LTO4G1', 'file_family': 'BACKFILL2', 'wrapper': 'cpio_odc', 'address': ('131.225.13.59', 7502), 'volume_family': 'cms.BACKFILL2.cpio_odc', 'file_family_width': 15}, 'outfilepath': '/pnfs/fnal.gov/usr/cms/WAX//11/store/backfill/2/data/IansMagicMushroomSoup-T0Test-AnalyzeThisAndGetAFreeBigFatHairyPhD/JetMonitor/RAW/v10_31_pre20replaythingy_v15/000/000/015/80F0E71A-45C6-DE11-A8B3-001D09F29597.root', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 25, 'delayed_dismount': None, 'curpri': 25}, 'file_size': 2099839175L, 'mover': {'max_buffer': 2621440000L, 'check_written_file': 600, 'max_failures': 5, 'library': 'CD-LTO4G1.library_manager', 'local_mover': 0, 'single_filemark': 1, 'max_consecutive_failures': 3, 'norestart': 'INQ', 'port': 7863, 'mc_device': '0,6,1,2', 'compression': 0, 'serial_num': '1310050821', 'update_interval_in_bound': 2, 'update_interval': 5, 'syslog_entry': 'sense|st[0-9]', 'status': ('ok', None), 'max_rate': 73400320.0, 'vendor_id': 'IBM', 'driver': 'FTTDriver', 'host': 'stkenmvr203a', 'max_time_in_state': 1200, 'device': 'stkenmvr203a:/dev/rmt/tps0d0n', 'name': 'LTO4_103.mover', 'send_stats': 1, 'product_id': 'ULTRIUM-TD4', 'statistics_path': '/tmp/enstore/enstore/LTO4_103.stat', 'mover_address': ('131.225.13.198', 7863), 'mount_delay': 15, 'check_first_written_file': 0, 'callback_addr': ('131.225.13.198', 57831), 'data_ip': 'stkenmvr203a', 'logname': 'LTO4_103MV', 'hostip': '131.225.13.198', 'do_cleaning': 'No', 'media_changer': 'SL8500G1.media_changer'}, 'ignore_fair_share': None, 'retry': None, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fnal.gov/usr/cms/WAX//11/store/backfill/2/data/IansMagicMushroomSoup-T0Test-AnalyzeThisAndGetAFreeBigFatHairyPhD/JetMonitor/RAW/v10_31_pre20replaythingy_v15/000/000/015/80F0E71A-45C6-DE11-A8B3-001D09F29597.root', 'uid': 0, 'uname': 'root', 'mtime': 1257141677, 'rmajor': 0, 'machine': ('Linux', 'cmsstor194.fnal.gov', '2.6.31', '#1 SMP Mon Sep 14 09:43:20 CDT 2009', 'x86_64'), 'sanity_size': 65536, 'type': 'cpio_odc', 'pstat': (16877, 126372832, 21L, 1, 9811, 5063, 512, 1257141604, 1257141604, 1257107163), 'mode_octal': '0100016', 'mode': 33188, 'size_bytes': 2099839175L, 'gname': 'root', 'gid': 0, 'fullname': '/storage/data1/write-pool-1/data/0007000000000000618C8FA8', 'inode': 126652328L, 'minor': 21}, 'version': 'v3_7d CVS $Revision: 1.922 $ encp', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/storage/data1/write-pool-1/data/0007000000000000618C8FA8', 'status': ('ok', None), 'outfile': '/pnfs/fnal.gov/usr/cms/WAX//11/store/backfill/2/data/IansMagicMushroomSoup-T0Test-AnalyzeThisAndGetAFreeBigFatHairyPhD/JetMonitor/RAW/v10_31_pre20replaythingy_v15/000/000/015/.(access)(0007000000000000618C8FA8)', 'fc': {'size': 2099839175L, 'external_label': 'VOR423', 'address': ('131.225.13.59', 7501)}, 'infilepath': '/storage/data1/write-pool-1/data/0007000000000000618C8FA8', 'at_the_top': 5, 'local_inode': 7339, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.207.19', 41677), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'lm_dequeued': 1257141678.6338661, 'encp_start_time': 1257141675.1245601, 'job_queued': 1257141677.1714289, 't0': 1257141675, 'in_queue': 1.4623761177062988}, 'user_level': 'admin', 'unique_id': 'cmsstor194.fnal.gov-1257141677-16956-0'} Thread client_connect_thread 00:01:18 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:01:18 cmsstor194.fnal.gov 016956 root I ENCP Control socket ('131.225.207.19', 41677) is connected to ('131.225.13.198', 57832) for cmsstor194.fnal.gov-1257141677-16956-0. elapsed=3.540sec 00:01:18 cmsstor194.fnal.gov 016956 root I ENCP Received callback ticket from mover LTO4_103.mover for transfer cmsstor194.fnal.gov-1257141677-16956-0. elapsed=3.550sec 00:01:18 cmsstor194.fnal.gov 016956 root I ENCP Atempting to connect data socket to mover at ('131.225.13.198', 57831). 00:01:18 cmsstor194.fnal.gov 016956 root I ENCP Connecting data socket to mover (('131.225.13.198', 57831)) with interface 131.225.207.19. elapsed=3.550sec 00:01:18 cmsstor194.fnal.gov 016956 root I ENCP Data socket ('131.225.207.19', 47706) is connected to ('131.225.13.198', 57831). elapsed=3.550sec 00:01:18 cmsstor194.fnal.gov 016956 root I ENCP Opening local file. 00:01:18 stkenmvr203a.fnal.gov 006116 root I LTO4_103MV tape position: target 38 current 38 0 Thread seek_thread 00:01:18 stkenmvr203a.fnal.gov 006116 root I LTO4_103MV write_tape starting, bytes_to_write=2099840000 Thread tape_thread 00:01:18 stkenmvr203a.fnal.gov 006116 root I LTO4_103MV Write starting. Tape VOR423 absolute location in blocks 914290 Thread tape_thread 00:01:18 stkensrv4n.fnal.gov 003689 enstore I SL8500G1MC MC Overflow: 4 mount VON584 0,5,1,15 00:01:18 stkenmvr148a.fnal.gov 006010 root E LTO4_48MV loadvol ('MEDIA_CHANGER_QUEUE_FULL', 0, 'retry later') Thread media_thread 00:01:18 stkenmvr148a.fnal.gov 006010 root I LTO4_48MV Media Changer returned MEDIA_CHANGER_QUEUE_FULL Thread media_thread 00:01:18 cmsstor194.fnal.gov 016956 root I ENCP pf_status = ('ok', None) skip_layer_cleanup = False 00:01:19 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:01:19 cmsstor197.fnal.gov 008600 root I ENCP Creating zero length output files. 00:01:19 cmsstor197.fnal.gov 008600 root I ENCP Sending /pnfs/fnal.gov/usr/cms/WAX//11/store/backfill/2/data/IansMagicMushroomSoup-T0Test-AnalyzeThisAndGetAFreeBigFatHairyPhD/JetMonitor/RAW/v10_31_pre20replaythingy_v15/000/000/015/6270F549-3DC6-DE11-AC53-000423D99A8E.root write request to LM: unique_id: cmsstor197.fnal.gov-1257141679-8600-0 inputfile: /storage/data1/write-pool-1/data/0007000000000000618C8FF0 outputfile: /pnfs/fnal.gov/usr/cms/WAX//11/store/backfill/2/data/IansMagicMushroomSoup-T0Test-AnalyzeThisAndGetAFreeBigFatHairyPhD/JetMonitor/RAW/v10_31_pre20replaythingy_v15/000/000/015/.(access)(0007000000000000618C8FF0) 00:01:19 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM MSG_TYPE=ADD_TO_LMQ request added to pending queue for storage group : cms 00:01:19 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM write Q'd /storage/data1/write-pool-1/data/0007000000000000618C8FF0 (cmsstor197.fnal.gov-1257141679-8600-0) -> /pnfs/fnal.gov/usr/cms/WAX//11/store/backfill/2/data/IansMagicMushroomSoup-T0Test-AnalyzeThisAndGetAFreeBigFatHairyPhD/JetMonitor/RAW/v10_31_pre20replaythingy_v15/000/000/015/6270F549-3DC6-DE11-AC53-000423D99A8E.root : library=CD-LTO4G1 family=BACKFILL2 requester:root volume_family:cms.BACKFILL2.cpio_odc 00:01:19 cmsstor197.fnal.gov 008600 root I ENCP Listening for control socket at: ('131.225.207.22', 54460) 00:01:19 cmsstor197.fnal.gov 008600 root I ENCP Waiting for mover to connect control socket at address: ('131.225.207.22', 54460) elapsed=2.210sec 00:01:19 stkensrv0n.fnal.gov 028762 enstore I VOLSRV Assigning volume VOR203 from storage group cms.BACKFILL2.cpio_odc to library CD-LTO4G1, volume family cms.BACKFILL2.cpio_odc 00:01:19 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM write_to_hsm work on vol=VOR203 mover=LTO4_47.mover requester:root 00:01:19 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM MSG_TYPE=ADD_TO_LMQ request added to work at movers queue for storage group : cms 00:01:19 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM IDLE:sending {'vc': {'storage_group': 'cms', 'library': 'CD-LTO4G1', 'file_family': 'BACKFILL2', 'wrapper': 'cpio_odc', 'address': ('131.225.13.59', 7502), 'volume_family': 'cms.BACKFILL2.cpio_odc', 'file_family_width': 15}, 'outfilepath': '/pnfs/fnal.gov/usr/cms/WAX//11/store/backfill/2/data/IansMagicMushroomSoup-T0Test-AnalyzeThisAndGetAFreeBigFatHairyPhD/JetMonitor/RAW/v10_31_pre20replaythingy_v15/000/000/015/6270F549-3DC6-DE11-AC53-000423D99A8E.root', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 25, 'delayed_dismount': None, 'curpri': 25}, 'file_size': 2107940588L, 'mover': 'LTO4_47.mover', 'ignore_fair_share': None, 'retry': None, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fnal.gov/usr/cms/WAX//11/store/backfill/2/data/IansMagicMushroomSoup-T0Test-AnalyzeThisAndGetAFreeBigFatHairyPhD/JetMonitor/RAW/v10_31_pre20replaythingy_v15/000/000/015/6270F549-3DC6-DE11-AC53-000423D99A8E.root', 'uid': 0, 'uname': 'root', 'type': 'cpio_odc', 'gid': 0, 'mtime': 1257141679, 'rmajor': 0, 'machine': ('Linux', 'cmsstor197.fnal.gov', '2.6.31', '#1 SMP Mon Sep 14 09:43:20 CDT 2009', 'x86_64'), 'sanity_size': 65536, 'pstat': (16877, 126372832, 21L, 1, 9811, 5063, 512, 1257141604, 1257141604, 1257107163), 'mode_octal': '0100016', 'mode': 33188, 'gname': 'root', 'size_bytes': 2107940588L, 'fullname': '/storage/data1/write-pool-1/data/0007000000000000618C8FF0', 'inode': 126652400L, 'minor': 21}, 'version': 'v3_7d CVS $Revision: 1.922 $ encp', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/storage/data1/write-pool-1/data/0007000000000000618C8FF0', 'status': ('ok', None), 'outfile': '/pnfs/fnal.gov/usr/cms/WAX//11/store/backfill/2/data/IansMagicMushroomSoup-T0Test-AnalyzeThisAndGetAFreeBigFatHairyPhD/JetMonitor/RAW/v10_31_pre20replaythingy_v15/000/000/015/.(access)(0007000000000000618C8FF0)', 'fc': {'size': 2107940588L, 'external_label': 'VOR203', 'address': ('131.225.13.59', 7501)}, 'infilepath': '/storage/data1/write-pool-1/data/0007000000000000618C8FF0', 'at_the_top': 2, 'local_inode': 3261, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.207.22', 54460), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'lm_dequeued': 1257141679.8039839, 'encp_start_time': 1257141677.5220981, 'job_queued': 1257141679.7054451, 't0': 1257141677, 'in_queue': 0.098471879959106445}, 'user_level': 'admin', 'unique_id': 'cmsstor197.fnal.gov-1257141679-8600-0'} to mover 00:01:19 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM can not remove from sg LTO4_47.mover VOR203 00:01:19 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM SG: {'exp-db': [('LTO4_107.mover', 'VON399'), ('LTO4_102.mover', 'VON218'), ('LTO4_108.mover', 'VON217'), ('LTO4_96.mover', 'VON705'), ('LTO4_40.mover', 'VON706')], 'cms': [('LTO4_100.mover', 'VOR424'), ('LTO4_49.mover', 'VOR381'), ('LTO4_51.mover', 'VOP202'), ('LTO4_110.mover', 'VOR407'), ('LTO4_104.mover', 'VOR382'), ('LTO4_44.mover', 'VOR426'), ('LTO4_101.mover', 'VOR413'), ('LTO4_109.mover', 'VOR198'), ('LTO4_41.mover', 'VOR202'), ('LTO4_45.mover', 'VOR385'), ('LTO4_43.mover', 'VOR416'), ('LTO4_46.mover', 'VOR200'), ('LTO4_103.mover', 'VOR423'), ('LTO4_98.mover', 'VOR201')], 'minos': [('LTO4_48.mover', 'VON584')]} 00:01:19 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM can not remove from vf LTO4_47.mover VOR203 00:01:19 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM VF: {'cms.NonCustodial.cpio_odc': [('LTO4_51.mover', 'VOP202')], 'minos.reco_far_dogwood1_bntp.cpio_odc': [('LTO4_48.mover', 'VON584')], 'exp-db.archive-d0-offline.cpio_odc': [('LTO4_107.mover', 'VON399')], 'exp-db.monthly-cdf-offline.cpio_odc': [('LTO4_102.mover', 'VON218'), ('LTO4_108.mover', 'VON217')], 'exp-db.monthly-d0-offline.cpio_odc': [('LTO4_96.mover', 'VON705'), ('LTO4_40.mover', 'VON706')], 'cms.BACKFILL2.cpio_odc': [('LTO4_100.mover', 'VOR424'), ('LTO4_49.mover', 'VOR381'), ('LTO4_110.mover', 'VOR407'), ('LTO4_104.mover', 'VOR382'), ('LTO4_44.mover', 'VOR426'), ('LTO4_101.mover', 'VOR413'), ('LTO4_109.mover', 'VOR198'), ('LTO4_41.mover', 'VOR202'), ('LTO4_45.mover', 'VOR385'), ('LTO4_43.mover', 'VOR416'), ('LTO4_46.mover', 'VOR200'), ('LTO4_103.mover', 'VOR423'), ('LTO4_98.mover', 'VOR201')]} 00:01:19 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:01:19 stkenmvr147a.fnal.gov 009702 root I LTO4_47MV WRITE_TO_HSM Thread MainThread 00:01:19 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:01:19 stkenmvr147a.fnal.gov 009702 root I LTO4_47MV SENDING {'vc': {'storage_group': 'cms', 'library': 'CD-LTO4G1', 'file_family': 'BACKFILL2', 'wrapper': 'cpio_odc', 'address': ('131.225.13.59', 7502), 'volume_family': 'cms.BACKFILL2.cpio_odc', 'file_family_width': 15}, 'outfilepath': '/pnfs/fnal.gov/usr/cms/WAX//11/store/backfill/2/data/IansMagicMushroomSoup-T0Test-AnalyzeThisAndGetAFreeBigFatHairyPhD/JetMonitor/RAW/v10_31_pre20replaythingy_v15/000/000/015/6270F549-3DC6-DE11-AC53-000423D99A8E.root', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 25, 'delayed_dismount': None, 'curpri': 25}, 'file_size': 2107940588L, 'mover': {'max_buffer': 2621440000L, 'check_written_file': 600, 'max_failures': 5, 'library': 'CD-LTO4G1.library_manager', 'local_mover': 0, 'dismount_delay': 30, 'max_consecutive_failures': 3, 'norestart': 'INQ', 'port': 7847, 'mc_device': '0,6,1,0', 'compression': 0, 'serial_num': '1310154418', 'update_interval_in_bound': 2, 'update_interval': 5, 'syslog_entry': 'sense|st[0-9]', 'status': ('ok', None), 'max_rate': 73400320.0, 'vendor_id': 'IBM', 'driver': 'FTTDriver', 'host': 'stkenmvr147a', 'max_time_in_state': 1200, 'device': 'stkenmvr147a:/dev/rmt/tps0d0n', 'name': 'LTO4_47.mover', 'send_stats': 1, 'product_id': 'ULTRIUM-TD4', 'statistics_path': '/tmp/enstore/enstore/LTO4_47.stat', 'single_filemark': 1, 'mount_delay': 15, 'mover_address': ('131.225.13.175', 7847), 'check_first_written_file': 0, 'callback_addr': ('131.225.13.175', 41529), 'data_ip': 'stkenmvr147a', 'logname': 'LTO4_47MV', 'hostip': '131.225.13.175', 'do_cleaning': 'No', 'media_changer': 'SL8500G1.media_changer', 'min_buffer': 67108864L}, 'ignore_fair_share': None, 'retry': None, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fnal.gov/usr/cms/WAX//11/store/backfill/2/data/IansMagicMushroomSoup-T0Test-AnalyzeThisAndGetAFreeBigFatHairyPhD/JetMonitor/RAW/v10_31_pre20replaythingy_v15/000/000/015/6270F549-3DC6-DE11-AC53-000423D99A8E.root', 'uid': 0, 'uname': 'root', 'mtime': 1257141679, 'rmajor': 0, 'machine': ('Linux', 'cmsstor197.fnal.gov', '2.6.31', '#1 SMP Mon Sep 14 09:43:20 CDT 2009', 'x86_64'), 'sanity_size': 65536, 'type': 'cpio_odc', 'pstat': (16877, 126372832, 21L, 1, 9811, 5063, 512, 1257141604, 1257141604, 1257107163), 'mode_octal': '0100016', 'mode': 33188, 'size_bytes': 2107940588L, 'gname': 'root', 'gid': 0, 'fullname': '/storage/data1/write-pool-1/data/0007000000000000618C8FF0', 'inode': 126652400L, 'minor': 21}, 'version': 'v3_7d CVS $Revision: 1.922 $ encp', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/storage/data1/write-pool-1/data/0007000000000000618C8FF0', 'status': ('ok', None), 'outfile': '/pnfs/fnal.gov/usr/cms/WAX//11/store/backfill/2/data/IansMagicMushroomSoup-T0Test-AnalyzeThisAndGetAFreeBigFatHairyPhD/JetMonitor/RAW/v10_31_pre20replaythingy_v15/000/000/015/.(access)(0007000000000000618C8FF0)', 'fc': {'size': 2107940588L, 'external_label': 'VOR203', 'address': ('131.225.13.59', 7501)}, 'infilepath': '/storage/data1/write-pool-1/data/0007000000000000618C8FF0', 'at_the_top': 2, 'local_inode': 3261, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.207.22', 54460), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'lm_dequeued': 1257141679.8039839, 'encp_start_time': 1257141677.5220981, 'job_queued': 1257141679.7054451, 't0': 1257141677, 'in_queue': 0.098471879959106445}, 'user_level': 'admin', 'unique_id': 'cmsstor197.fnal.gov-1257141679-8600-0'} Thread client_connect_thread 00:01:19 cmsstor197.fnal.gov 008600 root I ENCP Control socket ('131.225.207.22', 54460) is connected to ('131.225.13.175', 41530) for cmsstor197.fnal.gov-1257141679-8600-0. elapsed=2.330sec 00:01:19 cmsstor197.fnal.gov 008600 root I ENCP Received callback ticket from mover LTO4_47.mover for transfer cmsstor197.fnal.gov-1257141679-8600-0. elapsed=2.340sec 00:01:19 cmsstor197.fnal.gov 008600 root I ENCP Atempting to connect data socket to mover at ('131.225.13.175', 41529). 00:01:19 cmsstor197.fnal.gov 008600 root I ENCP Connecting data socket to mover (('131.225.13.175', 41529)) with interface 131.225.207.22. elapsed=2.340sec 00:01:19 cmsstor197.fnal.gov 008600 root I ENCP Data socket ('131.225.207.22', 47278) is connected to ('131.225.13.175', 41529). elapsed=2.340sec 00:01:19 cmsstor197.fnal.gov 008600 root I ENCP Opening local file. 00:01:19 stkenmvr147a.fnal.gov 009702 root I LTO4_47MV MSG_TYPE=MC_LOAD_REQ mounting VOR203 ULTRIUM-TD4 00:01:19 Thread media_thread 00:01:19 stkensrv4n.fnal.gov 003689 enstore I SL8500G1MC MC Overflow: 4 mount VOR203 0,6,1,0 00:01:19 stkenmvr147a.fnal.gov 009702 root E LTO4_47MV loadvol ('MEDIA_CHANGER_QUEUE_FULL', 0, 'retry later') Thread media_thread 00:01:19 stkenmvr147a.fnal.gov 009702 root I LTO4_47MV Media Changer returned MEDIA_CHANGER_QUEUE_FULL Thread media_thread 00:01:20 stkensrv4n.fnal.gov 003689 enstore I SL8500G1MC MC Overflow: 4 mount VOR381 0,5,1,3 00:01:20 stkenmvr149a.fnal.gov 006318 root E LTO4_49MV loadvol ('MEDIA_CHANGER_QUEUE_FULL', 0, 'retry later') Thread media_thread 00:01:20 stkenmvr149a.fnal.gov 006318 root I LTO4_49MV Media Changer returned MEDIA_CHANGER_QUEUE_FULL Thread media_thread 00:01:20 cmsstor197.fnal.gov 008600 root I ENCP pf_status = ('ok', None) skip_layer_cleanup = False 00:01:21 stkenmvr141a.fnal.gov 013963 root I LTO4_41MV tape position: target 188 current 0 1 Thread seek_thread 00:01:22 stkensrv4n.fnal.gov 003689 enstore I SL8500G1MC MC Overflow: 4 mount VOP202 0,5,1,12 00:01:22 stkenmvr151a.fnal.gov 013228 root E LTO4_51MV loadvol ('MEDIA_CHANGER_QUEUE_FULL', 0, 'retry later') Thread media_thread 00:01:22 stkenmvr151a.fnal.gov 013228 root I LTO4_51MV Media Changer returned MEDIA_CHANGER_QUEUE_FULL Thread media_thread 00:01:25 cmsstor184.fnal.gov 019235 root I ENCP Version: v3_7d CVS $Revision: 1.922 $ encp OS: Linux 2.6.31 x86_64 Release: Scientific Linux SL release 5.2 (Boron) Rocks release 5.1 (V.I) User: root(0) Group: root(0) Euser: root(0) Egroup: root(0) Library: CD-LTO4G1 Storage Group: cms File Family: BACKFILL2 FF Wrapper: cpio_odc FF Width: 15 Current working directory: cmsstor184.fnal.gov:/ Command line: /home/enstore/enstore/bin/encp --verbose 4 --threaded --ecrc --resubmit-timeout 1800 --age-time 30 --delpri 1 --priority 25 --pnfs-mount /pnfs/fs --shortcut --override-path /pnfs/fnal.gov/usr/cms/WAX//11/store/backfill/2/data/IansMagicMushroomSoup-T0Test-AnalyzeThisAndGetAFreeBigFatHairyPhD/JetMonitor/RAW/v10_31_pre20replaythingy_v15/000/000/015/DC55D2AF-3BC6-DE11-811C-001D09F29597.root --put-cache 0007000000000000618C8F90 /storage/data3/write-pool-3/data/0007000000000000618C8F90 00:01:25 cmsstor184.fnal.gov 019235 root I ENCP Creating zero length output files. 00:01:25 cmsstor184.fnal.gov 019235 root I ENCP Sending /pnfs/fnal.gov/usr/cms/WAX//11/store/backfill/2/data/IansMagicMushroomSoup-T0Test-AnalyzeThisAndGetAFreeBigFatHairyPhD/JetMonitor/RAW/v10_31_pre20replaythingy_v15/000/000/015/DC55D2AF-3BC6-DE11-811C-001D09F29597.root write request to LM: unique_id: cmsstor184.fnal.gov-1257141685-19235-0 inputfile: /storage/data3/write-pool-3/data/0007000000000000618C8F90 outputfile: /pnfs/fnal.gov/usr/cms/WAX//11/store/backfill/2/data/IansMagicMushroomSoup-T0Test-AnalyzeThisAndGetAFreeBigFatHairyPhD/JetMonitor/RAW/v10_31_pre20replaythingy_v15/000/000/015/.(access)(0007000000000000618C8F90) 00:01:25 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM MSG_TYPE=ADD_TO_LMQ request added to pending queue for storage group : cms 00:01:25 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM write Q'd /storage/data3/write-pool-3/data/0007000000000000618C8F90 (cmsstor184.fnal.gov-1257141685-19235-0) -> /pnfs/fnal.gov/usr/cms/WAX//11/store/backfill/2/data/IansMagicMushroomSoup-T0Test-AnalyzeThisAndGetAFreeBigFatHairyPhD/JetMonitor/RAW/v10_31_pre20replaythingy_v15/000/000/015/DC55D2AF-3BC6-DE11-811C-001D09F29597.root : library=CD-LTO4G1 family=BACKFILL2 requester:root volume_family:cms.BACKFILL2.cpio_odc 00:01:25 cmsstor184.fnal.gov 019235 root I ENCP Listening for control socket at: ('131.225.207.5', 60037) 00:01:25 cmsstor184.fnal.gov 019235 root I ENCP Waiting for mover to connect control socket at address: ('131.225.207.5', 60037) elapsed=0.720sec 00:01:25 stkensrv0n.fnal.gov 028762 enstore I VOLSRV Assigning volume VOR422 from storage group cms.BACKFILL2.cpio_odc to library CD-LTO4G1, volume family cms.BACKFILL2.cpio_odc 00:01:25 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM write_to_hsm work on vol=VOR422 mover=LTO4_106.mover requester:root 00:01:25 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM MSG_TYPE=ADD_TO_LMQ request added to work at movers queue for storage group : cms 00:01:25 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM IDLE:sending {'vc': {'storage_group': 'cms', 'library': 'CD-LTO4G1', 'file_family': 'BACKFILL2', 'wrapper': 'cpio_odc', 'address': ('131.225.13.59', 7502), 'volume_family': 'cms.BACKFILL2.cpio_odc', 'file_family_width': 15}, 'outfilepath': '/pnfs/fnal.gov/usr/cms/WAX//11/store/backfill/2/data/IansMagicMushroomSoup-T0Test-AnalyzeThisAndGetAFreeBigFatHairyPhD/JetMonitor/RAW/v10_31_pre20replaythingy_v15/000/000/015/DC55D2AF-3BC6-DE11-811C-001D09F29597.root', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 25, 'delayed_dismount': None, 'curpri': 25}, 'file_size': 2101444715L, 'mover': 'LTO4_106.mover', 'ignore_fair_share': None, 'retry': None, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fnal.gov/usr/cms/WAX//11/store/backfill/2/data/IansMagicMushroomSoup-T0Test-AnalyzeThisAndGetAFreeBigFatHairyPhD/JetMonitor/RAW/v10_31_pre20replaythingy_v15/000/000/015/DC55D2AF-3BC6-DE11-811C-001D09F29597.root', 'uid': 0, 'uname': 'root', 'type': 'cpio_odc', 'gid': 0, 'mtime': 1257141685, 'rmajor': 0, 'machine': ('Linux', 'cmsstor184.fnal.gov', '2.6.31', '#1 SMP Mon Sep 14 09:43:20 CDT 2009', 'x86_64'), 'sanity_size': 65536, 'pstat': (16877, 126372832, 21L, 1, 9811, 5063, 512, 1257141604, 1257141604, 1257107163), 'mode_octal': '0100016', 'mode': 33188, 'gname': 'root', 'size_bytes': 2101444715L, 'fullname': '/storage/data3/write-pool-3/data/0007000000000000618C8F90', 'inode': 126652304L, 'minor': 21}, 'version': 'v3_7d CVS $Revision: 1.922 $ encp', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/storage/data3/write-pool-3/data/0007000000000000618C8F90', 'status': ('ok', None), 'outfile': '/pnfs/fnal.gov/usr/cms/WAX//11/store/backfill/2/data/IansMagicMushroomSoup-T0Test-AnalyzeThisAndGetAFreeBigFatHairyPhD/JetMonitor/RAW/v10_31_pre20replaythingy_v15/000/000/015/.(access)(0007000000000000618C8F90)', 'fc': {'size': 2101444715L, 'external_label': 'VOR422', 'address': ('131.225.13.59', 7501)}, 'infilepath': '/storage/data3/write-pool-3/data/0007000000000000618C8F90', 'at_the_top': 2, 'local_inode': 5338, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.207.5', 60037), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'lm_dequeued': 1257141685.879281, 'encp_start_time': 1257141685.0744131, 'job_queued': 1257141685.7810881, 't0': 1257141685, 'in_queue': 0.098131895065307617}, 'user_level': 'admin', 'unique_id': 'cmsstor184.fnal.gov-1257141685-19235-0'} to mover 00:01:25 stkenmvr206a.fnal.gov 026766 root I LTO4_106MV WRITE_TO_HSM Thread MainThread 00:01:25 stkenmvr206a.fnal.gov 026766 root I LTO4_106MV SENDING {'vc': {'storage_group': 'cms', 'library': 'CD-LTO4G1', 'file_family': 'BACKFILL2', 'wrapper': 'cpio_odc', 'address': ('131.225.13.59', 7502), 'volume_family': 'cms.BACKFILL2.cpio_odc', 'file_family_width': 15}, 'outfilepath': '/pnfs/fnal.gov/usr/cms/WAX//11/store/backfill/2/data/IansMagicMushroomSoup-T0Test-AnalyzeThisAndGetAFreeBigFatHairyPhD/JetMonitor/RAW/v10_31_pre20replaythingy_v15/000/000/015/DC55D2AF-3BC6-DE11-811C-001D09F29597.root', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 25, 'delayed_dismount': None, 'curpri': 25}, 'file_size': 2101444715L, 'mover': {'max_buffer': 2621440000L, 'check_written_file': 600, 'max_failures': 5, 'library': 'CD-LTO4G1.library_manager', 'local_mover': 0, 'single_filemark': 1, 'max_consecutive_failures': 3, 'norestart': 'INQ', 'port': 7866, 'mc_device': '0,6,1,8', 'compression': 0, 'serial_num': '1310152619', 'update_interval_in_bound': 2, 'update_interval': 5, 'syslog_entry': 'sense|st[0-9]', 'status': ('ok', None), 'max_rate': 73400320.0, 'vendor_id': 'IBM', 'driver': 'FTTDriver', 'host': 'stkenmvr206a', 'max_time_in_state': 1200, 'device': 'stkenmvr206a:/dev/rmt/tps0d0n', 'name': 'LTO4_106.mover', 'send_stats': 1, 'product_id': 'ULTRIUM-TD4', 'statistics_path': '/tmp/enstore/enstore/LTO4_106.stat', 'mover_address': ('131.225.13.202', 7866), 'mount_delay': 15, 'check_first_written_file': 0, 'callback_addr': ('131.225.13.202', 40271), 'data_ip': 'stkenmvr206a', 'logname': 'LTO4_106MV', 'hostip': '131.225.13.202', 'do_cleaning': 'No', 'media_changer': 'SL8500G1.media_changer'}, 'ignore_fair_share': None, 'retry': None, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fnal.gov/usr/cms/WAX//11/store/backfill/2/data/IansMagicMushroomSoup-T0Test-AnalyzeThisAndGetAFreeBigFatHairyPhD/JetMonitor/RAW/v10_31_pre20replaythingy_v15/000/000/015/DC55D2AF-3BC6-DE11-811C-001D09F29597.root', 'uid': 0, 'uname': 'root', 'mtime': 1257141685, 'rmajor': 0, 'machine': ('Linux', 'cmsstor184.fnal.gov', '2.6.31', '#1 SMP Mon Sep 14 09:43:20 CDT 2009', 'x86_64'), 'sanity_size': 65536, 'type': 'cpio_odc', 'pstat': (16877, 126372832, 21L, 1, 9811, 5063, 512, 1257141604, 1257141604, 1257107163), 'mode_octal': '0100016', 'mode': 33188, 'size_bytes': 2101444715L, 'gname': 'root', 'gid': 0, 'fullname': '/storage/data3/write-pool-3/data/0007000000000000618C8F90', 'inode': 126652304L, 'minor': 21}, 'version': 'v3_7d CVS $Revision: 1.922 $ encp', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/storage/data3/write-pool-3/data/0007000000000000618C8F90', 'status': ('ok', None), 'outfile': '/pnfs/fnal.gov/usr/cms/WAX//11/store/backfill/2/data/IansMagicMushroomSoup-T0Test-AnalyzeThisAndGetAFreeBigFatHairyPhD/JetMonitor/RAW/v10_31_pre20replaythingy_v15/000/000/015/.(access)(0007000000000000618C8F90)', 'fc': {'size': 2101444715L, 'external_label': 'VOR422', 'address': ('131.225.13.59', 7501)}, 'infilepath': '/storage/data3/write-pool-3/data/0007000000000000618C8F90', 'at_the_top': 2, 'local_inode': 5338, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.207.5', 60037), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'lm_dequeued': 1257141685.879281, 'encp_start_time': 1257141685.0744131, 'job_queued': 1257141685.7810881, 't0': 1257141685, 'in_queue': 0.098131895065307617}, 'user_level': 'admin', 'unique_id': 'cmsstor184.fnal.gov-1257141685-19235-0'} Thread client_connect_thread 00:01:25 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM can not remove from sg LTO4_106.mover VOR422 00:01:25 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM SG: {'exp-db': [('LTO4_107.mover', 'VON399'), ('LTO4_102.mover', 'VON218'), ('LTO4_108.mover', 'VON217'), ('LTO4_96.mover', 'VON705'), ('LTO4_40.mover', 'VON706')], 'cms': [('LTO4_100.mover', 'VOR424'), ('LTO4_49.mover', 'VOR381'), ('LTO4_51.mover', 'VOP202'), ('LTO4_110.mover', 'VOR407'), ('LTO4_104.mover', 'VOR382'), ('LTO4_44.mover', 'VOR426'), ('LTO4_101.mover', 'VOR413'), ('LTO4_109.mover', 'VOR198'), ('LTO4_41.mover', 'VOR202'), ('LTO4_45.mover', 'VOR385'), ('LTO4_43.mover', 'VOR416'), ('LTO4_46.mover', 'VOR200'), ('LTO4_103.mover', 'VOR423'), ('LTO4_98.mover', 'VOR201'), ('LTO4_47.mover', 'VOR203')], 'minos': [('LTO4_48.mover', 'VON584')]} 00:01:25 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM can not remove from vf LTO4_106.mover VOR422 00:01:25 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM VF: {'cms.NonCustodial.cpio_odc': [('LTO4_51.mover', 'VOP202')], 'minos.reco_far_dogwood1_bntp.cpio_odc': [('LTO4_48.mover', 'VON584')], 'exp-db.archive-d0-offline.cpio_odc': [('LTO4_107.mover', 'VON399')], 'exp-db.monthly-cdf-offline.cpio_odc': [('LTO4_102.mover', 'VON218'), ('LTO4_108.mover', 'VON217')], 'exp-db.monthly-d0-offline.cpio_odc': [('LTO4_96.mover', 'VON705'), ('LTO4_40.mover', 'VON706')], 'cms.BACKFILL2.cpio_odc': [('LTO4_100.mover', 'VOR424'), ('LTO4_49.mover', 'VOR381'), ('LTO4_110.mover', 'VOR407'), ('LTO4_104.mover', 'VOR382'), ('LTO4_44.mover', 'VOR426'), ('LTO4_101.mover', 'VOR413'), ('LTO4_109.mover', 'VOR198'), ('LTO4_41.mover', 'VOR202'), ('LTO4_45.mover', 'VOR385'), ('LTO4_43.mover', 'VOR416'), ('LTO4_46.mover', 'VOR200'), ('LTO4_103.mover', 'VOR423'), ('LTO4_98.mover', 'VOR201'), ('LTO4_47.mover', 'VOR203')]} 00:01:25 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:01:25 stkensrv4n.fnal.gov 005795 enstore I 9940BLM 00:01:25 cmsstor184.fnal.gov 019235 root I ENCP Control socket ('131.225.207.5', 60037) is connected to ('131.225.13.202', 40272) for cmsstor184.fnal.gov-1257141685-19235-0. elapsed=0.830sec 00:01:25 cmsstor184.fnal.gov 019235 root I ENCP Received callback ticket from mover LTO4_106.mover for transfer cmsstor184.fnal.gov-1257141685-19235-0. elapsed=0.840sec 00:01:25 cmsstor184.fnal.gov 019235 root I ENCP Atempting to connect data socket to mover at ('131.225.13.202', 40271). 00:01:25 cmsstor184.fnal.gov 019235 root I ENCP Connecting data socket to mover (('131.225.13.202', 40271)) with interface 131.225.207.5. elapsed=0.840sec 00:01:25 cmsstor184.fnal.gov 019235 root I ENCP Data socket ('131.225.207.5', 44497) is connected to ('131.225.13.202', 40271). elapsed=0.840sec 00:01:25 cmsstor184.fnal.gov 019235 root I ENCP Opening local file. 00:01:26 stkenmvr206a.fnal.gov 026766 root I LTO4_106MV MSG_TYPE=MC_LOAD_REQ mounting VOR422 ULTRIUM-TD4 00:01:25 Thread media_thread 00:01:26 stkensrv4n.fnal.gov 003689 enstore I SL8500G1MC MC Overflow: 4 mount VOR422 0,6,1,8 00:01:26 stkenmvr206a.fnal.gov 026766 root E LTO4_106MV loadvol ('MEDIA_CHANGER_QUEUE_FULL', 0, 'retry later') Thread media_thread 00:01:26 stkenmvr206a.fnal.gov 026766 root I LTO4_106MV Media Changer returned MEDIA_CHANGER_QUEUE_FULL Thread media_thread 00:01:26 cmsstor184.fnal.gov 019235 root I ENCP pf_status = ('ok', None) skip_layer_cleanup = False 00:01:26 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:01:26 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:01:26 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:01:27 cmsstor82.fnal.gov 029584 root I ENCP Version: v3_7d CVS $Revision: 1.922 $ encp OS: Linux 2.6.31 x86_64 Release: Rocks release 5.1 (V.I) Scientific Linux SL release 5.2 (Boron) User: root(0) Group: root(0) Euser: root(0) Egroup: root(0) Library: CD-LTO4G1 Storage Group: cms File Family: BACKFILL2 FF Wrapper: cpio_odc FF Width: 15 Current working directory: cmsstor82.fnal.gov:/ Command line: /home/enstore/enstore/bin/encp --verbose 4 --threaded --ecrc --resubmit-timeout 1800 --age-time 30 --delpri 1 --priority 25 --pnfs-mount /pnfs/fs --shortcut --override-path /pnfs/fnal.gov/usr/cms/WAX//11/store/backfill/2/data/IansMagicMushroomSoup-T0Test-AnalyzeThisAndGetAFreeBigFatHairyPhD/JetMonitor/RAW/v10_31_pre20replaythingy_v15/000/000/015/4652C74E-41C6-DE11-90E3-000423DD2F34.root --put-cache 0007000000000000618C8FF8 /storage/data2/write-pool-2/data/0007000000000000618C8FF8 00:01:27 cmsstor82.fnal.gov 029584 root I ENCP Creating zero length output files. 00:01:27 cmsstor82.fnal.gov 029584 root I ENCP Sending /pnfs/fnal.gov/usr/cms/WAX//11/store/backfill/2/data/IansMagicMushroomSoup-T0Test-AnalyzeThisAndGetAFreeBigFatHairyPhD/JetMonitor/RAW/v10_31_pre20replaythingy_v15/000/000/015/4652C74E-41C6-DE11-90E3-000423DD2F34.root write request to LM: unique_id: cmsstor82.fnal.gov-1257141687-29584-0 inputfile: /storage/data2/write-pool-2/data/0007000000000000618C8FF8 outputfile: /pnfs/fnal.gov/usr/cms/WAX//11/store/backfill/2/data/IansMagicMushroomSoup-T0Test-AnalyzeThisAndGetAFreeBigFatHairyPhD/JetMonitor/RAW/v10_31_pre20replaythingy_v15/000/000/015/.(access)(0007000000000000618C8FF8) 00:01:27 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM MSG_TYPE=ADD_TO_LMQ request added to pending queue for storage group : cms 00:01:27 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM write Q'd /storage/data2/write-pool-2/data/0007000000000000618C8FF8 (cmsstor82.fnal.gov-1257141687-29584-0) -> /pnfs/fnal.gov/usr/cms/WAX//11/store/backfill/2/data/IansMagicMushroomSoup-T0Test-AnalyzeThisAndGetAFreeBigFatHairyPhD/JetMonitor/RAW/v10_31_pre20replaythingy_v15/000/000/015/4652C74E-41C6-DE11-90E3-000423DD2F34.root : library=CD-LTO4G1 family=BACKFILL2 requester:root volume_family:cms.BACKFILL2.cpio_odc 00:01:27 cmsstor82.fnal.gov 029584 root I ENCP Listening for control socket at: ('131.225.189.82', 41450) 00:01:27 cmsstor82.fnal.gov 029584 root I ENCP Waiting for mover to connect control socket at address: ('131.225.189.82', 41450) elapsed=1.090sec 00:01:28 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:01:28 stkensrv4n.fnal.gov 003689 enstore I SL8500G1MC MC Overflow: 4 mount VOR382 0,6,1,13 00:01:28 stkenmvr204a.fnal.gov 006261 root E LTO4_104MV loadvol ('MEDIA_CHANGER_QUEUE_FULL', 0, 'retry later') Thread media_thread 00:01:28 stkenmvr204a.fnal.gov 006261 root I LTO4_104MV Media Changer returned MEDIA_CHANGER_QUEUE_FULL Thread media_thread 00:01:28 stkensrv4n.fnal.gov 003689 enstore I SL8500G1MC MC Overflow: 4 mount VON584 0,5,1,15 00:01:28 stkenmvr148a.fnal.gov 006010 root E LTO4_48MV loadvol ('MEDIA_CHANGER_QUEUE_FULL', 0, 'retry later') Thread media_thread 00:01:28 stkenmvr148a.fnal.gov 006010 root I LTO4_48MV Media Changer returned MEDIA_CHANGER_QUEUE_FULL Thread media_thread 00:01:29 stkensrv4n.fnal.gov 003689 enstore I SL8500G1MC MC Overflow: 4 mount VOR203 0,6,1,0 00:01:29 stkenmvr147a.fnal.gov 009702 root E LTO4_47MV loadvol ('MEDIA_CHANGER_QUEUE_FULL', 0, 'retry later') Thread media_thread 00:01:29 stkenmvr147a.fnal.gov 009702 root I LTO4_47MV Media Changer returned MEDIA_CHANGER_QUEUE_FULL Thread media_thread 00:01:29 cmsstor182.fnal.gov 000859 root I ENCP File /storage/data1/write-pool-1/data/0007000000000000618C88F8 transfered for cmsstor182.fnal.gov-1257141503-859-0 in 17.7491290569 seconds. 00:01:30 stkensrv4n.fnal.gov 003689 enstore I SL8500G1MC MC Overflow: 4 mount VOR381 0,5,1,3 00:01:30 stkenmvr149a.fnal.gov 006318 root E LTO4_49MV loadvol ('MEDIA_CHANGER_QUEUE_FULL', 0, 'retry later') Thread media_thread 00:01:30 stkenmvr149a.fnal.gov 006318 root I LTO4_49MV Media Changer returned MEDIA_CHANGER_QUEUE_FULL Thread media_thread 00:01:32 cmsstor63.fnal.gov 005003 root I ENCP File /storage/data2/write-pool-2/data/0007000000000000618C8FE8 transfered for cmsstor63.fnal.gov-1257141672-5003-0 in 17.8283429146 seconds. 00:01:32 stkensrv4n.fnal.gov 003689 enstore I SL8500G1MC MC Overflow: 4 mount VOP202 0,5,1,12 00:01:32 stkenmvr151a.fnal.gov 013228 root E LTO4_51MV loadvol ('MEDIA_CHANGER_QUEUE_FULL', 0, 'retry later') Thread media_thread 00:01:32 stkenmvr151a.fnal.gov 013228 root I LTO4_51MV Media Changer returned MEDIA_CHANGER_QUEUE_FULL Thread media_thread 00:01:33 stkensrv4n.fnal.gov 005828 enstore I LTO4F1LM 00:01:33 stkenmvr144a.fnal.gov 006329 root I LTO4_44MV written bytes 2096337408/2096337408, blocks 15994 header 292 trailer 255 Thread tape_thread 00:01:33 stkenmvr146a.fnal.gov 005990 root I LTO4_46MV written bytes 2098472960/2098472960, blocks 16011 header 292 trailer 90 Thread tape_thread 00:01:34 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:01:34 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:01:34 stkenmvr144a.fnal.gov 006329 root I LTO4_44MV filemarks written. Tape VOR426 absolute location in blocks 909510 Thread tape_thread 00:01:34 stkenmvr144a.fnal.gov 006329 root I LTO4_44MV write_tape timing:pull 0.204950809479 write 10.2006316185 crc_check 8.46033358574 freespace 0.198161840439 block_write 19.0640778542 idle 0.313277006149 Thread tape_thread 00:01:34 stkenmvr144a.fnal.gov 006329 root I LTO4_44MV drive stats after write. Tape VOR426 position 39 block 0 block_size 0 bloc_loc 909510 tot_blocks 762939000 BOT 0 read_err 0 write_err 0 bytes 2096337408 block_write_tot 19.0640778542 tape_rate 109962696.546 Thread tape_thread 00:01:34 stkenmvr144a.fnal.gov 006329 root I LTO4_44MV remaining bytes info in DB 668061696000 estimated 665965358592 reported from drive 666119168000 position 40 Thread tape_thread 00:01:34 stkenmvr144a.fnal.gov 006329 root I LTO4_44MV new bitfile request {'uid': 0, 'external_label': 'VOR426', 'gid': 0, 'location_cookie': '0000_000000000_0000039', 'complete_crc': 794028598L, 'sanity_cookie': (65536L, 1048993388L), 'size': 2096336861L} Thread tape_thread 00:01:35 stkenmvr144a.fnal.gov 006329 root I LTO4_44MV set remaining: VOR426 666119168000 0000_000000000_0000040 Thread tape_thread 00:01:35 last message repeated 1 times 00:01:35 stkensrv0n.fnal.gov 028762 enstore I VOLSRV Updating update volume set last_access = '2009-11-02 00:01:35', eod_cookie = '0000_000000000_0000040', non_del_files = '39', remaining_bytes = 666119168000 where label = 'VOR426'; 00:01:35 stkenmvr144a.fnal.gov 006329 root I LTO4_44MV transfer complete volume=VOR426 location=40 Thread tape_thread 00:01:35 last message repeated 1 times 00:01:35 stkensrv0n.fnal.gov 028762 enstore I VOLSRV Updating update volume set sum_wr_access = '39', modification_time = '2009-11-02 00:01:35' where label = 'VOR426'; 00:01:35 stkensrv4n.fnal.gov 005795 enstore I 9940BLM 00:01:35 stkenmvr144a.fnal.gov 006329 root I LTO4_44MV Sending done to client: {'vc': {'comment': '', 'storage_group': 'cms', 'blocksize': 131072, 'declared': 1255358936.0, 'library': 'CD-LTO4G1', 'si_time': [0.0, 1246295206.0], 'user_inhibit': ['none', 'none'], 'system_inhibit': ['none', 'none'], 'external_label': 'VOR426', 'wrapper': 'cpio_odc', 'remaining_bytes': 668061696000L, 'sum_mounts': 76, 'sum_rd_access': 0, 'file_family_width': 15, 'last_access': 1257141379.0, 'media_type': 'LTO4', 'status': ('ok', None), 'eod_cookie': '0000_000000000_0000039', 'capacity_bytes': 858993459200L, 'non_del_files': 38, 'sum_wr_err': 0, 'sum_wr_access': 38, 'file_family': 'BACKFILL2', 'address': ('131.225.13.59', 7502), 'volume_family': 'cms.BACKFILL2.cpio_odc', 'modification_time': 1257141379.0, 'write_protected': 'n', 'sum_rd_err': 0, 'first_access': 1257134618.0}, 'outfilepath': '/pnfs/fnal.gov/usr/cms/WAX//11/store/backfill/2/data/IansMagicMushroomSoup-T0Test-AnalyzeThisAndGetAFreeBigFatHairyPhD/JetMonitor/RAW/v10_31_pre20replaythingy_v15/000/000/015/2896CA85-40C6-DE11-AD17-000423D99BF2.root', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 25, 'delayed_dismount': None, 'curpri': 25}, 'file_size': 2096336861L, 'mover': {'max_buffer': 2621440000L, 'check_written_file': 600, 'max_failures': 5, 'library': 'CD-LTO4G1.library_manager', 'local_mover': 0, 'dismount_delay': 30, 'max_consecutive_failures': 3, 'norestart': 'INQ', 'port': 7844, 'mc_device': '0,6,1,15', 'compression': 0, 'serial_num': '9310123734', 'update_interval_in_bound': 2, 'update_interval': 5, 'syslog_entry': 'sense|st[0-9]', 'status': ('ok', None), 'max_rate': 73400320.0, 'vendor_id': 'IBM', 'driver': 'FTTDriver', 'host': 'stkenmvr144a', 'max_time_in_state': 1200, 'device': 'stkenmvr144a:/dev/rmt/tps0d0n', 'name': 'LTO4_44.mover', 'send_stats': 1, 'product_id': 'ULTRIUM-TD4', 'statistics_path': '/tmp/enstore/enstore/LTO4_44.stat', 'single_filemark': 1, 'mount_delay': 15, 'mover_address': ('131.225.13.172', 7844), 'check_first_written_file': 0, 'callback_addr': ('131.225.13.172', 42876), 'data_ip': 'stkenmvr144a', 'logname': 'LTO4_44MV', 'hostip': '131.225.13.172', 'do_cleaning': 'No', 'media_changer': 'SL8500G1.media_changer', 'min_buffer': 67108864L}, 'ignore_fair_share': None, 'retry': None, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fnal.gov/usr/cms/WAX//11/store/backfill/2/data/IansMagicMushroomSoup-T0Test-AnalyzeThisAndGetAFreeBigFatHairyPhD/JetMonitor/RAW/v10_31_pre20replaythingy_v15/000/000/015/2896CA85-40C6-DE11-AD17-000423D99BF2.root', 'uid': 0, 'uname': 'root', 'mtime': 1257141503, 'rmajor': 0, 'machine': ('Linux', 'cmsstor182.fnal.gov', '2.6.31', '#1 SMP Mon Sep 14 09:43:20 CDT 2009', 'x86_64'), 'sanity_size': 65536, 'type': 'cpio_odc', 'pstat': (16877, 126372832, 23L, 1, 9811, 5063, 512, 1257141485, 1257141485, 1257107163), 'mode_octal': '0100016', 'mode': 33188, 'size_bytes': 2096336861L, 'gname': 'root', 'gid': 0, 'fullname': '/storage/data1/write-pool-1/data/0007000000000000618C88F8', 'inode': 126650616L, 'minor': 23}, 'version': 'v3_7d CVS $Revision: 1.922 $ encp', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/storage/data1/write-pool-1/data/0007000000000000618C88F8', 'status': ('ok', None), 'outfile': '/pnfs/fnal.gov/usr/cms/WAX//11/store/backfill/2/data/IansMagicMushroomSoup-T0Test-AnalyzeThisAndGetAFreeBigFatHairyPhD/JetMonitor/RAW/v10_31_pre20replaythingy_v15/000/000/015/.(access)(0007000000000000618C88F8)', 'fc': {'uid': 0, 'external_label': 'VOR426', 'gid': 0, 'location_cookie': '0000_000000000_0000039', 'complete_crc': 794028598L, 'bfid': 'CDMS125714169400000', 'sanity_cookie': (65536L, 1048993388L), 'size': 2096336861L}, 'infilepath': '/storage/data1/write-pool-1/data/0007000000000000618C88F8', 'at_the_top': 19, 'local_inode': 5621, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.207.1', 57283), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'encp_start_time': 1257141503.002347, 'seek_time': 152.37340712547302, 'transfer_t(trunc. 4382) Thread tape_thread 00:01:35 stkenmvr144a.fnal.gov 006329 root I LTO4_44MV Done is sent Thread tape_thread 00:01:35 stkenmvr144a.fnal.gov 006329 root I LTO4_44MV maybe_clean Thread tape_thread 00:01:35 cmsstor182.fnal.gov 000859 root I ENCP Received final dialog for cmsstor182.fnal.gov-1257141503-859-0. elapsed=192.110sec 00:01:35 cmsstor182.fnal.gov 000859 root I ENCP leaving transfer_file(). 00:01:35 stkenmvr144a.fnal.gov 006329 root I LTO4_44MV returned from maybe_clean Thread tape_thread 00:01:35 stkenmvr146a.fnal.gov 005990 root I LTO4_46MV filemarks written. Tape VOR200 absolute location in blocks 4766541 Thread tape_thread 00:01:35 stkenmvr146a.fnal.gov 005990 root I LTO4_46MV write_tape timing:pull 0.17788028717 write 8.08617377281 crc_check 8.25835490227 freespace 0.182514667511 block_write 16.7049236298 idle 0.302164077759 Thread tape_thread 00:01:35 stkenmvr146a.fnal.gov 005990 root I LTO4_46MV drive stats after write. Tape VOR200 position 199 block 0 block_size 0 bloc_loc 4766541 tot_blocks 762939000 BOT 0 read_err 0 write_err 0 bytes 2098472960 block_write_tot 16.7049236298 tape_rate 125620027.155 Thread tape_thread 00:01:35 stkenmvr146a.fnal.gov 005990 root I LTO4_46MV remaining bytes info in DB 198424576000 estimated 196326103040 reported from drive 196481024000 position 200 Thread tape_thread 00:01:35 stkenmvr146a.fnal.gov 005990 root I LTO4_46MV new bitfile request {'uid': 0, 'external_label': 'VOR200', 'gid': 0, 'location_cookie': '0000_000000000_0000199', 'complete_crc': 2302166596L, 'sanity_cookie': (65536L, 2147991349L), 'size': 2098472578L} Thread tape_thread 00:01:35 stkenmvr146a.fnal.gov 005990 root I LTO4_46MV set remaining: VOR200 196481024000 0000_000000000_0000200 Thread tape_thread 00:01:35 last message repeated 1 times 00:01:35 cmsstor182.fnal.gov 000859 root I ENCP pf_status = ('ok', None) skip_layer_cleanup = False 00:01:35 stkensrv0n.fnal.gov 028762 enstore I VOLSRV Updating update volume set last_access = '2009-11-02 00:01:35', eod_cookie = '0000_000000000_0000200', non_del_files = '199', remaining_bytes = 196481024000 where label = 'VOR200'; 00:01:35 stkenmvr146a.fnal.gov 005990 root I LTO4_46MV transfer complete volume=VOR200 location=200 Thread tape_thread 00:01:35 last message repeated 1 times 00:01:35 stkensrv0n.fnal.gov 028762 enstore I VOLSRV Updating update volume set sum_wr_access = '199', modification_time = '2009-11-02 00:01:35' where label = 'VOR200'; 00:01:35 stkenmvr146a.fnal.gov 005990 root I LTO4_46MV Sending done to client: {'vc': {'comment': '', 'storage_group': 'cms', 'blocksize': 131072, 'declared': 1255358745.0, 'library': 'CD-LTO4G1', 'si_time': [0.0, 1238581927.0], 'user_inhibit': ['none', 'none'], 'system_inhibit': ['none', 'none'], 'external_label': 'VOR200', 'wrapper': 'cpio_odc', 'remaining_bytes': 198424576000L, 'sum_mounts': 91, 'sum_rd_access': 0, 'file_family_width': 15, 'last_access': 1257141673.0, 'media_type': 'LTO4', 'status': ('ok', None), 'eod_cookie': '0000_000000000_0000199', 'capacity_bytes': 858993459200L, 'non_del_files': 198, 'sum_wr_err': 0, 'sum_wr_access': 198, 'file_family': 'BACKFILL2', 'address': ('131.225.13.59', 7502), 'volume_family': 'cms.BACKFILL2.cpio_odc', 'modification_time': 1257141673.0, 'write_protected': 'n', 'sum_rd_err': 0, 'first_access': 1257103365.0}, 'outfilepath': '/pnfs/fnal.gov/usr/cms/WAX//11/store/backfill/2/data/IansMagicMushroomSoup-T0Test-AnalyzeThisAndGetAFreeBigFatHairyPhD/JetMonitor/RAW/v10_31_pre20replaythingy_v15/000/000/015/8A623D3F-40C6-DE11-B943-001D09F29533.root', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 25, 'delayed_dismount': None, 'curpri': 25}, 'file_size': 2098472578L, 'mover': {'max_buffer': 2621440000L, 'check_written_file': 600, 'max_failures': 5, 'library': 'CD-LTO4G1.library_manager', 'local_mover': 0, 'dismount_delay': 30, 'max_consecutive_failures': 3, 'norestart': 'INQ', 'port': 7846, 'mc_device': '0,6,1,12', 'compression': 0, 'serial_num': '9311032672', 'update_interval_in_bound': 2, 'update_interval': 5, 'syslog_entry': 'sense|st[0-9]', 'status': ('ok', None), 'max_rate': 73400320.0, 'vendor_id': 'IBM', 'driver': 'FTTDriver', 'host': 'stkenmvr146a', 'max_time_in_state': 1200, 'device': 'stkenmvr146a:/dev/rmt/tps0d0n', 'name': 'LTO4_46.mover', 'send_stats': 1, 'product_id': 'ULTRIUM-TD4', 'statistics_path': '/tmp/enstore/enstore/LTO4_46.stat', 'single_filemark': 1, 'mount_delay': 15, 'mover_address': ('131.225.13.174', 7846), 'check_first_written_file': 0, 'callback_addr': ('131.225.13.174', 56681), 'data_ip': 'stkenmvr146a', 'logname': 'LTO4_46MV', 'hostip': '131.225.13.174', 'do_cleaning': 'No', 'media_changer': 'SL8500G1.media_changer', 'min_buffer': 67108864L}, 'ignore_fair_share': None, 'retry': None, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fnal.gov/usr/cms/WAX//11/store/backfill/2/data/IansMagicMushroomSoup-T0Test-AnalyzeThisAndGetAFreeBigFatHairyPhD/JetMonitor/RAW/v10_31_pre20replaythingy_v15/000/000/015/8A623D3F-40C6-DE11-B943-001D09F29533.root', 'uid': 0, 'uname': 'root', 'mtime': 1257141672, 'rmajor': 0, 'machine': ('Linux', 'cmsstor63.fnal.gov', '2.6.31', '#1 SMP Mon Sep 14 09:43:20 CDT 2009', 'x86_64'), 'sanity_size': 65536, 'type': 'cpio_odc', 'pstat': (16877, 126372832, 21L, 1, 9811, 5063, 512, 1257141604, 1257141604, 1257107163), 'mode_octal': '0100016', 'mode': 33188, 'size_bytes': 2098472578L, 'gname': 'root', 'gid': 0, 'fullname': '/storage/data2/write-pool-2/data/0007000000000000618C8FE8', 'inode': 126652392L, 'minor': 21}, 'version': 'v3_7d CVS $Revision: 1.922 $ encp', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/storage/data2/write-pool-2/data/0007000000000000618C8FE8', 'status': ('ok', None), 'outfile': '/pnfs/fnal.gov/usr/cms/WAX//11/store/backfill/2/data/IansMagicMushroomSoup-T0Test-AnalyzeThisAndGetAFreeBigFatHairyPhD/JetMonitor/RAW/v10_31_pre20replaythingy_v15/000/000/015/.(access)(0007000000000000618C8FE8)', 'fc': {'uid': 0, 'external_label': 'VOR200', 'gid': 0, 'location_cookie': '0000_000000000_0000199', 'complete_crc': 2302166596L, 'bfid': 'CDMS125714169500000', 'sanity_cookie': (65536L, 2147991349L), 'size': 2098472578L}, 'infilepath': '/storage/data2/write-pool-2/data/0007000000000000618C8FE8', 'at_the_top': 9, 'local_inode': 1073742074, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.204.239', 47703), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'encp_start_time': 1257141671.8277791, 'seek_time': 0.035510778427124023,(trunc. 4430) Thread tape_thread 00:01:35 stkenmvr146a.fnal.gov 005990 root I LTO4_46MV Done is sent Thread tape_thread 00:01:35 stkenmvr146a.fnal.gov 005990 root I LTO4_46MV maybe_clean Thread tape_thread 00:01:35 stkenmvr146a.fnal.gov 005990 root I LTO4_46MV returned from maybe_clean Thread tape_thread 00:01:35 stkensrv4n.fnal.gov 005357 enstore E LTO4G1LM MOVER ERROR RQ {'status': ('ok', None), 'returned_work': None, 'library': None, 'mover_type': '', 'current_location': '0000_000000000_0000000', 'ip_map': '', 'operation': None, 'read_only': 0, 'current_time': 1257141695.298929, 'error_source': None, 'external_label': None, 'state': 'OFFLINE', 'r_a': (('131.225.13.194', 54880), 11096L, '131.225.13.194-54880-1256586929.509013-4866'), 'library_list': [('CD-LTO4G1.library_manager', ('131.225.13.4', 8541))], 'current_priority': (None, None), 'transfer_deficiency': 1, 'time_in_state': 554764.93887090683, 'address': ('131.225.13.194', 7860), 'volume_family': None, 'work': 'mover_error', 'mover': 'LTO4_99.mover', 'volume_status': (['none', 'none'], ['none', 'none']), 'unique_id': None} 00:01:35 cmsstor63.fnal.gov 005003 root I ENCP Received final dialog for cmsstor63.fnal.gov-1257141672-5003-0. elapsed=23.520sec 00:01:35 stkensrv0n.fnal.gov 028796 enstore I FILSRV Updating update file set deleted = 'n', pnfs_path = '/pnfs/fnal.gov/usr/cms/WAX//11/store/backfill/2/data/IansMagicMushroomSoup-T0Test-AnalyzeThisAndGetAFreeBigFatHairyPhD/JetMonitor/RAW/v10_31_pre20replaythingy_v15/000/000/015/2896CA85-40C6-DE11-AD17-000423D99BF2.root', drive = 'stkenmvr144a:/dev/rmt/tps0d0n:9310123734', pnfs_id = '0007000000000000618C88F8' where bfid = 'CDMS125714169400000'; 00:01:35 cmsstor63.fnal.gov 005003 root I ENCP leaving transfer_file(). 00:01:35 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:01:35 cmsstor182.fnal.gov 000859 root I ENCP MSG_TYPE=ENCP_XFER write_to_hsm /storage/data1/write-pool-1/data/0007000000000000618C88F8 -> /pnfs/fnal.gov/usr/cms/WAX//11/store/backfill/2/data/IansMagicMushroomSoup-T0Test-AnalyzeThisAndGetAFreeBigFatHairyPhD/JetMonitor/RAW/v10_31_pre20replaythingy_v15/000/000/015/2896CA85-40C6-DE11-AD17-000423D99BF2.root: 2096336861 bytes copied to VOR426 at 87.3 MB/S (98.1 MB/S network) (94.7 MB/S drive) (113 MB/S disk) mover=LTO4_44.mover drive_id=ULTRIUM-TD4 drive_sn=9310123734 drive_vendor=IBM elapsed=192.46 {'storage_group': 'cms', 'encp_crc': 794028598L, 'disk_rate': 112.66893097872862, 'drive_rate': 94.723318089140335, 'driver': 'FTTDriver', 'transfer_rate': 87.312791523873386, 'mover_interface': 'stkenmvr144a', 'overall_rate': 11.382593548869311, 'media_changer': 'SL8500G1.media_changer', 'network_rate': 87.312791523873386, 'encp_ip': '131.225.207.1', 'unique_id': 'cmsstor182.fnal.gov-1257141503-859-0'} 00:01:35 cmsstor182.fnal.gov 000859 root I ENCP File /pnfs/fnal.gov/usr/cms/WAX//11/store/backfill/2/data/IansMagicMushroomSoup-T0Test-AnalyzeThisAndGetAFreeBigFatHairyPhD/JetMonitor/RAW/v10_31_pre20replaythingy_v15/000/000/015/2896CA85-40C6-DE11-AD17-000423D99BF2.root copied successfully. 00:01:35 cmsstor182.fnal.gov 000859 root I ENCP Completed transferring 2096336861 bytes in 1 files in 192.459516048 sec. Overall rate = 11.4 MB/sec. Transfer rate = 87.3 MB/sec. Network rate = 98.1 MB/sec. Drive rate = 94.7 MB/sec. Disk rate = 113 MB/sec. Exit status = 0. 00:01:35 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM write_to_hsm next work on vol=VOR200 mover=LTO4_46.mover requester:root 00:01:35 cmsstor63.fnal.gov 005003 root I ENCP pf_status = ('ok', None) skip_layer_cleanup = False 00:01:35 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM HAVE_BOUND:sending write_to_hsm /pnfs/fnal.gov/usr/cms/WAX//11/store/backfill/2/data/IansMagicMushroomSoup-T0Test-AnalyzeThisAndGetAFreeBigFatHairyPhD/JetMonitor/RAW/v10_31_pre20replaythingy_v15/000/000/015/4652C74E-41C6-DE11-90E3-000423DD2F34.root to mover LTO4_46.mover ('131.225.13.174', 7846) DEL_DISM None 00:01:35 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM MSG_TYPE=ADD_TO_LMQ request added to work at movers queue for storage group : cms 00:01:35 last message repeated 1 times 00:01:35 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM HAVE_BOUND:sending {'vc': {'storage_group': 'cms', 'external_label': 'VOR200', 'library': 'CD-LTO4G1', 'file_family': 'BACKFILL2', 'wrapper': 'cpio_odc', 'address': ('131.225.13.59', 7502), 'volume_family': 'cms.BACKFILL2.cpio_odc', 'file_family_width': 15}, 'outfilepath': '/pnfs/fnal.gov/usr/cms/WAX//11/store/backfill/2/data/IansMagicMushroomSoup-T0Test-AnalyzeThisAndGetAFreeBigFatHairyPhD/JetMonitor/RAW/v10_31_pre20replaythingy_v15/000/000/015/4652C74E-41C6-DE11-90E3-000423DD2F34.root', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 25, 'delayed_dismount': None, 'curpri': 25}, 'file_size': 2102707869L, 'mover': 'LTO4_46.mover', 'ignore_fair_share': None, 'retry': None, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fnal.gov/usr/cms/WAX//11/store/backfill/2/data/IansMagicMushroomSoup-T0Test-AnalyzeThisAndGetAFreeBigFatHairyPhD/JetMonitor/RAW/v10_31_pre20replaythingy_v15/000/000/015/4652C74E-41C6-DE11-90E3-000423DD2F34.root', 'uid': 0, 'uname': 'root', 'type': 'cpio_odc', 'gid': 0, 'mtime': 1257141687, 'rmajor': 0, 'machine': ('Linux', 'cmsstor82.fnal.gov', '2.6.31', '#1 SMP Mon Sep 14 09:43:20 CDT 2009', 'x86_64'), 'sanity_size': 65536, 'pstat': (16877, 126372832, 21L, 1, 9811, 5063, 512, 1257141604, 1257141604, 1257107163), 'mode_octal': '0100016', 'mode': 33188, 'gname': 'root', 'size_bytes': 2102707869L, 'fullname': '/storage/data2/write-pool-2/data/0007000000000000618C8FF8', 'inode': 126652408L, 'minor': 21}, 'version': 'v3_7d CVS $Revision: 1.922 $ encp', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/storage/data2/write-pool-2/data/0007000000000000618C8FF8', 'status': ('ok', None), 'outfile': '/pnfs/fnal.gov/usr/cms/WAX//11/store/backfill/2/data/IansMagicMushroomSoup-T0Test-AnalyzeThisAndGetAFreeBigFatHairyPhD/JetMonitor/RAW/v10_31_pre20replaythingy_v15/000/000/015/.(access)(0007000000000000618C8FF8)', 'fc': {'size': 2102707869L, 'external_label': 'VOR200', 'address': ('131.225.13.59', 7501)}, 'infilepath': '/storage/data2/write-pool-2/data/0007000000000000618C8FF8', 'at_the_top': 27, 'local_inode': 2147484221L, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.189.82', 41450), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'lm_dequeued': 1257141695.4821191, 'encp_start_time': 1257141686.3958831, 'job_queued': 1257141687.452914, 't0': 1257141686, 'in_queue': 8.0290689468383789}, 'user_level': 'admin', 'unique_id': 'cmsstor82.fnal.gov-1257141687-29584-0'} to mover 00:01:35 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:01:35 stkenmvr146a.fnal.gov 005990 root I LTO4_46MV WRITE_TO_HSM Thread MainThread 00:01:35 stkenmvr146a.fnal.gov 005990 root I LTO4_46MV SENDING {'vc': {'storage_group': 'cms', 'external_label': 'VOR200', 'library': 'CD-LTO4G1', 'file_family': 'BACKFILL2', 'wrapper': 'cpio_odc', 'address': ('131.225.13.59', 7502), 'volume_family': 'cms.BACKFILL2.cpio_odc', 'file_family_width': 15}, 'outfilepath': '/pnfs/fnal.gov/usr/cms/WAX//11/store/backfill/2/data/IansMagicMushroomSoup-T0Test-AnalyzeThisAndGetAFreeBigFatHairyPhD/JetMonitor/RAW/v10_31_pre20replaythingy_v15/000/000/015/4652C74E-41C6-DE11-90E3-000423DD2F34.root', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 25, 'delayed_dismount': None, 'curpri': 25}, 'file_size': 2102707869L, 'mover': {'max_buffer': 2621440000L, 'check_written_file': 600, 'max_failures': 5, 'library': 'CD-LTO4G1.library_manager', 'local_mover': 0, 'dismount_delay': 30, 'max_consecutive_failures': 3, 'norestart': 'INQ', 'port': 7846, 'mc_device': '0,6,1,12', 'compression': 0, 'serial_num': '9311032672', 'update_interval_in_bound': 2, 'update_interval': 5, 'syslog_entry': 'sense|st[0-9]', 'status': ('ok', None), 'max_rate': 73400320.0, 'vendor_id': 'IBM', 'driver': 'FTTDriver', 'host': 'stkenmvr146a', 'max_time_in_state': 1200, 'device': 'stkenmvr146a:/dev/rmt/tps0d0n', 'name': 'LTO4_46.mover', 'send_stats': 1, 'product_id': 'ULTRIUM-TD4', 'statistics_path': '/tmp/enstore/enstore/LTO4_46.stat', 'single_filemark': 1, 'mount_delay': 15, 'mover_address': ('131.225.13.174', 7846), 'check_first_written_file': 0, 'callback_addr': ('131.225.13.174', 56683), 'data_ip': 'stkenmvr146a', 'logname': 'LTO4_46MV', 'hostip': '131.225.13.174', 'do_cleaning': 'No', 'media_changer': 'SL8500G1.media_changer', 'min_buffer': 67108864L}, 'ignore_fair_share': None, 'retry': None, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fnal.gov/usr/cms/WAX//11/store/backfill/2/data/IansMagicMushroomSoup-T0Test-AnalyzeThisAndGetAFreeBigFatHairyPhD/JetMonitor/RAW/v10_31_pre20replaythingy_v15/000/000/015/4652C74E-41C6-DE11-90E3-000423DD2F34.root', 'uid': 0, 'uname': 'root', 'mtime': 1257141687, 'rmajor': 0, 'machine': ('Linux', 'cmsstor82.fnal.gov', '2.6.31', '#1 SMP Mon Sep 14 09:43:20 CDT 2009', 'x86_64'), 'sanity_size': 65536, 'type': 'cpio_odc', 'pstat': (16877, 126372832, 21L, 1, 9811, 5063, 512, 1257141604, 1257141604, 1257107163), 'mode_octal': '0100016', 'mode': 33188, 'size_bytes': 2102707869L, 'gname': 'root', 'gid': 0, 'fullname': '/storage/data2/write-pool-2/data/0007000000000000618C8FF8', 'inode': 126652408L, 'minor': 21}, 'version': 'v3_7d CVS $Revision: 1.922 $ encp', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/storage/data2/write-pool-2/data/0007000000000000618C8FF8', 'status': ('ok', None), 'outfile': '/pnfs/fnal.gov/usr/cms/WAX//11/store/backfill/2/data/IansMagicMushroomSoup-T0Test-AnalyzeThisAndGetAFreeBigFatHairyPhD/JetMonitor/RAW/v10_31_pre20replaythingy_v15/000/000/015/.(access)(0007000000000000618C8FF8)', 'fc': {'size': 2102707869L, 'external_label': 'VOR200', 'address': ('131.225.13.59', 7501)}, 'infilepath': '/storage/data2/write-pool-2/data/0007000000000000618C8FF8', 'at_the_top': 27, 'local_inode': 2147484221L, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.189.82', 41450), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'lm_dequeued': 1257141695.4821191, 'encp_start_time': 1257141686.3958831, 'job_queued': 1257141687.452914, 't0': 1257141686, 'in_queue': 8.0290689468383789}, 'user_level': 'admin', 'unique_id': 'cmsstor82.fnal.gov-1257141687-29584-0'} Thread client_connect_thread 00:01:35 cmsstor82.fnal.gov 029584 root I ENCP Control socket ('131.225.189.82', 41450) is connected to ('131.225.13.174', 56684) for cmsstor82.fnal.gov-1257141687-29584-0. elapsed=9.190sec 00:01:35 cmsstor82.fnal.gov 029584 root I ENCP Received callback ticket from mover LTO4_46.mover for transfer cmsstor82.fnal.gov-1257141687-29584-0. elapsed=9.190sec 00:01:35 cmsstor82.fnal.gov 029584 root I ENCP Atempting to connect data socket to mover at ('131.225.13.174', 56683). 00:01:35 cmsstor82.fnal.gov 029584 root I ENCP Connecting data socket to mover (('131.225.13.174', 56683)) with interface 131.225.189.82. elapsed=9.190sec 00:01:35 cmsstor82.fnal.gov 029584 root I ENCP Data socket ('131.225.189.82', 49295) is connected to ('131.225.13.174', 56683). elapsed=9.190sec 00:01:35 cmsstor82.fnal.gov 029584 root I ENCP Opening local file. 00:01:35 stkensrv0n.fnal.gov 028796 enstore I FILSRV Updating update file set deleted = 'n', pnfs_path = '/pnfs/fnal.gov/usr/cms/WAX//11/store/backfill/2/data/IansMagicMushroomSoup-T0Test-AnalyzeThisAndGetAFreeBigFatHairyPhD/JetMonitor/RAW/v10_31_pre20replaythingy_v15/000/000/015/8A623D3F-40C6-DE11-B943-001D09F29533.root', drive = 'stkenmvr146a:/dev/rmt/tps0d0n:9311032672', pnfs_id = '0007000000000000618C8FE8' where bfid = 'CDMS125714169500000'; 00:01:35 stkenmvr146a.fnal.gov 005990 root I LTO4_46MV tape position: target 200 current 200 0 Thread seek_thread 00:01:35 stkenmvr146a.fnal.gov 005990 root I LTO4_46MV write_tape starting, bytes_to_write=2102708736 Thread tape_thread 00:01:35 cmsstor82.fnal.gov 029584 root I ENCP pf_status = ('ok', None) skip_layer_cleanup = False 00:01:35 stkenmvr146a.fnal.gov 005990 root I LTO4_46MV Write starting. Tape VOR200 absolute location in blocks 4766541 Thread tape_thread 00:01:35 cmsstor63.fnal.gov 005003 root I ENCP MSG_TYPE=ENCP_XFER write_to_hsm /storage/data2/write-pool-2/data/0007000000000000618C8FE8 -> /pnfs/fnal.gov/usr/cms/WAX//11/store/backfill/2/data/IansMagicMushroomSoup-T0Test-AnalyzeThisAndGetAFreeBigFatHairyPhD/JetMonitor/RAW/v10_31_pre20replaythingy_v15/000/000/015/8A623D3F-40C6-DE11-B943-001D09F29533.root: 2098472578 bytes copied to VOR200 at 98 MB/S (78.8 MB/S network) (107 MB/S drive) (112 MB/S disk) mover=LTO4_46.mover drive_id=ULTRIUM-TD4 drive_sn=9311032672 drive_vendor=IBM elapsed=23.932 {'storage_group': 'cms', 'encp_crc': 2302166596L, 'disk_rate': 112.29010545227491, 'drive_rate': 106.83836878411397, 'driver': 'FTTDriver', 'transfer_rate': 97.958849020065529, 'mover_interface': 'stkenmvr146a', 'overall_rate': 94.252599368847427, 'media_changer': 'SL8500G1.media_changer', 'network_rate': 97.958849020065529, 'encp_ip': '131.225.204.239', 'unique_id': 'cmsstor63.fnal.gov-1257141672-5003-0'} 00:01:35 cmsstor63.fnal.gov 005003 root I ENCP File /pnfs/fnal.gov/usr/cms/WAX//11/store/backfill/2/data/IansMagicMushroomSoup-T0Test-AnalyzeThisAndGetAFreeBigFatHairyPhD/JetMonitor/RAW/v10_31_pre20replaythingy_v15/000/000/015/8A623D3F-40C6-DE11-B943-001D09F29533.root copied successfully. 00:01:35 cmsstor63.fnal.gov 005003 root I ENCP Completed transferring 2098472578 bytes in 1 files in 23.9348599911 sec. Overall rate = 94.3 MB/sec. Transfer rate = 98 MB/sec. Network rate = 78.8 MB/sec. Drive rate = 107 MB/sec. Disk rate = 112 MB/sec. Exit status = 0. 00:01:35 stkendca20a.fnal.gov 032240 root I ENCP File /diskb/write-pool-2/data/00300000000000000619A558 transfered for stkendca20a.fnal.gov-1257141043-32240-0 in 62.1605231762 seconds. 00:01:35 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:01:35 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:01:35 stkensrv4n.fnal.gov 003689 enstore I SL8500G1MC MC Overflow: 4 mount VOR422 0,6,1,8 00:01:35 stkenmvr206a.fnal.gov 026766 root E LTO4_106MV loadvol ('MEDIA_CHANGER_QUEUE_FULL', 0, 'retry later') Thread media_thread 00:01:35 stkenmvr206a.fnal.gov 026766 root I LTO4_106MV Media Changer returned MEDIA_CHANGER_QUEUE_FULL Thread media_thread 00:01:36 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:01:36 cmsstor194.fnal.gov 016956 root I ENCP File /storage/data1/write-pool-1/data/0007000000000000618C8FA8 transfered for cmsstor194.fnal.gov-1257141677-16956-0 in 17.7592489719 seconds. 00:01:36 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:01:37 stkensrv4n.fnal.gov 030262 enstore I SL8500G1MC dismount VOLUME 0,6,1,3 force => 0,Dismount: Forced dismount of VOR385 from 0, 6, 1, 3 00:01:37 stkensrv4n.fnal.gov 003689 enstore I SL8500G1MC FINISHED dismount VOR385 0,6,1,3 returned ('ok', 0, 'dismount VOLUME 0,6,1,3 force => 0,Dismount: Forced dismount of VOR385 from 0, 6, 1, 3', '', '') 00:01:37 stkenmvr145a.fnal.gov 022974 root I LTO4_45MV dismounted VOR385 ULTRIUM-TD4 00:01:37 Thread media_thread 00:01:37 stkensrv4n.fnal.gov 030274 enstore I SL8500G1MC dismount VOLUME 0,6,1,11 force => 0,Dismount: Forced dismount of VOR424 from 0, 6, 1,11 00:01:37 stkensrv4n.fnal.gov 003689 enstore I SL8500G1MC FINISHED dismount VOR424 0,6,1,11 returned ('ok', 0, 'dismount VOLUME 0,6,1,11 force => 0,Dismount: Forced dismount of VOR424 from 0, 6, 1,11', '', '') 00:01:37 stkenmvr200a.fnal.gov 005076 root I LTO4_100MV dismounted VOR424 ULTRIUM-TD4 00:01:37 Thread media_thread 00:01:37 stkenmvr202a.fnal.gov 006121 root I LTO4_102MV written bytes 2147475968/2147475968, blocks 16384 header 190 trailer 322 Thread tape_thread 00:01:37 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:01:38 stkensrv4n.fnal.gov 003689 enstore I SL8500G1MC PROCESSING mount VOR382 0,6,1,13 00:01:38 stkensrv4n.fnal.gov 003689 enstore I SL8500G1MC PROCESSING mount VON584 0,5,1,15 00:01:39 stkenmvr202a.fnal.gov 006121 root I LTO4_102MV filemarks written. Tape VON218 absolute location in blocks 4954384 Thread tape_thread 00:01:39 stkenmvr202a.fnal.gov 006121 root I LTO4_102MV write_tape timing:pull 0.215412139893 write 11.2347285748 crc_check 9.8612806797 freespace 0.150342464447 block_write 21.4617638588 idle 0.282424926758 Thread tape_thread 00:01:39 stkenmvr202a.fnal.gov 006121 root I LTO4_102MV drive stats after write. Tape VON218 position 331 block 0 block_size 0 bloc_loc 4954384 tot_blocks 762939000 BOT 0 read_err 0 write_err 0 bytes 2147475968 block_write_tot 21.4617638588 tape_rate 100060553.37 Thread tape_thread 00:01:39 stkenmvr202a.fnal.gov 006121 root I LTO4_102MV remaining bytes info in DB 173529088000 estimated 171381612032 reported from drive 171526144000 position 332 Thread tape_thread 00:01:39 stkenmvr202a.fnal.gov 006121 root I LTO4_102MV new bitfile request {'uid': 0, 'external_label': 'VON218', 'gid': 0, 'location_cookie': '0000_000000000_0000331', 'complete_crc': 1811615136L, 'sanity_cookie': (65536L, 844419595L), 'size': 2147475456L} Thread tape_thread 00:01:39 stkenmvr202a.fnal.gov 006121 root I LTO4_102MV set remaining: VON218 171526144000 0000_000000000_0000332 Thread tape_thread 00:01:39 last message repeated 1 times 00:01:39 stkensrv0n.fnal.gov 028762 enstore I VOLSRV Updating update volume set last_access = '2009-11-02 00:01:39', eod_cookie = '0000_000000000_0000332', non_del_files = '377', remaining_bytes = 171526144000 where label = 'VON218'; 00:01:39 stkenmvr202a.fnal.gov 006121 root I LTO4_102MV transfer complete volume=VON218 location=332 Thread tape_thread 00:01:39 last message repeated 1 times 00:01:39 stkensrv0n.fnal.gov 028762 enstore I VOLSRV Updating update volume set sum_wr_access = '331', modification_time = '2009-11-02 00:01:39' where label = 'VON218'; 00:01:39 stkenmvr202a.fnal.gov 006121 root I LTO4_102MV Sending done to client: {'vc': {'comment': '', 'storage_group': 'exp-db', 'blocksize': 131072, 'declared': 1240352603.0, 'library': 'CD-LTO4G1', 'si_time': [0.0, 0.0], 'user_inhibit': ['none', 'none'], 'system_inhibit': ['none', 'none'], 'external_label': 'VON218', 'wrapper': 'cpio_odc', 'remaining_bytes': 173529088000L, 'sum_mounts': 61, 'sum_rd_access': 0, 'file_family_width': 2, 'last_access': 1257141632.0, 'media_type': 'LTO4', 'status': ('ok', None), 'eod_cookie': '0000_000000000_0000331', 'capacity_bytes': 858993459200L, 'non_del_files': 376, 'sum_wr_err': 0, 'sum_wr_access': 330, 'file_family': 'monthly-cdf-offline', 'address': ('131.225.13.59', 7502), 'volume_family': 'exp-db.monthly-cdf-offline.cpio_odc', 'modification_time': 1257141632.0, 'write_protected': 'n', 'sum_rd_err': 0, 'first_access': 1241271535.0}, 'outfilepath': '/pnfs/fnal.gov/usr/exp-db/monthly/cdf-offline/cdfofpr2/2009/11-November/01/channel01/CDFOFPR2_t701801352_s30277_p5', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 1, 'delayed_dismount': None, 'curpri': 1}, 'file_size': 2147475456L, 'mover': {'max_buffer': 2621440000L, 'check_written_file': 600, 'max_failures': 5, 'library': 'CD-LTO4G1.library_manager', 'local_mover': 0, 'single_filemark': 1, 'max_consecutive_failures': 3, 'norestart': 'INQ', 'port': 7862, 'mc_device': '0,6,1,14', 'compression': 0, 'serial_num': '1310051194', 'update_interval_in_bound': 2, 'update_interval': 5, 'syslog_entry': 'sense|st[0-9]', 'status': ('ok', None), 'max_rate': 73400320.0, 'vendor_id': 'IBM', 'driver': 'FTTDriver', 'host': 'stkenmvr202a', 'max_time_in_state': 1200, 'device': 'stkenmvr202a:/dev/rmt/tps0d0n', 'name': 'LTO4_102.mover', 'send_stats': 1, 'product_id': 'ULTRIUM-TD4', 'statistics_path': '/tmp/enstore/enstore/LTO4_102.stat', 'mover_address': ('131.225.13.197', 7862), 'mount_delay': 15, 'check_first_written_file': 0, 'callback_addr': ('131.225.13.197', 57455), 'data_ip': 'stkenmvr202a', 'logname': 'LTO4_102MV', 'hostip': '131.225.13.197', 'do_cleaning': 'No', 'media_changer': 'SL8500G1.media_changer'}, 'ignore_fair_share': None, 'retry': None, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fnal.gov/usr/exp-db/monthly/cdf-offline/cdfofpr2/2009/11-November/01/channel01/CDFOFPR2_t701801352_s30277_p5', 'uid': 0, 'uname': 'root', 'mtime': 1257141043, 'rmajor': 0, 'machine': ('Linux', 'stkendca20a.fnal.gov', '2.6.9-89.0.9.ELsmp', '#1 SMP Mon Aug 24 13:47:43 CDT 2009', 'i686'), 'sanity_size': 65536, 'type': 'cpio_odc', 'pstat': (16877, 820663368L, 20L, 1, 1602, 2752, 512L, 1257130346, 1257130346, 1254398469), 'mode_octal': '0100016', 'mode': 33188, 'size_bytes': 2147475456L, 'gname': 'root', 'gid': 0, 'fullname': '/diskb/write-pool-2/data/00300000000000000619A558', 'inode': 806987096L, 'minor': 20}, 'version': 'v3_8b CVS $Revision: 1.922 $ ', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/diskb/write-pool-2/data/00300000000000000619A558', 'status': ('ok', None), 'outfile': '/pnfs/fnal.gov/usr/exp-db/monthly/cdf-offline/cdfofpr2/2009/11-November/01/channel01/.(access)(00300000000000000619A558)', 'fc': {'uid': 0, 'external_label': 'VON218', 'gid': 0, 'location_cookie': '0000_000000000_0000331', 'complete_crc': 1811615136L, 'bfid': 'CDMS125714169900000', 'sanity_cookie': (65536L, 844419595L), 'size': 2147475456L}, 'infilepath': '/diskb/write-pool-2/data/00300000000000000619A558', 'at_the_top': 314, 'local_inode': 271282376L, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.13.12', 57125), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'encp_start_time': 1257141041.101377, 'seek_time': 0.030984163284301758, 'transfer_time': 66.026563882827759, 't0': 1257141041, 'mount_time': 0.016980886459350586, 'lm_dequeued': 1257141633.4799759, 'job_queued': 1257141043.5475941, 'drive_call_time': 11.020430326461792, 'in_queue': 589.93210387229919, 'drive_transfer_time': 23.507187843322754}, 'user_level': 'user2', 'unique_id': 'stkendca20a.fnal.go(trunc. 4102) Thread tape_thread 00:01:39 stkenmvr202a.fnal.gov 006121 root I LTO4_102MV Done is sent Thread tape_thread 00:01:39 stkenmvr202a.fnal.gov 006121 root I LTO4_102MV maybe_clean Thread tape_thread 00:01:39 stkenmvr202a.fnal.gov 006121 root I LTO4_102MV returned from maybe_clean Thread tape_thread 00:01:39 stkendca20a.fnal.gov 032240 root I ENCP Received final dialog for stkendca20a.fnal.gov-1257141043-32240-0. elapsed=658.530sec 00:01:39 stkendca20a.fnal.gov 032240 root I ENCP leaving transfer_file(). 00:01:39 stkendca20a.fnal.gov 032240 root I ENCP pf_status = ('ok', None) skip_layer_cleanup = False 00:01:39 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:01:39 stkensrv4n.fnal.gov 003689 enstore I SL8500G1MC MC Overflow: 4 mount VOR203 0,6,1,0 00:01:39 stkenmvr147a.fnal.gov 009702 root E LTO4_47MV loadvol ('MEDIA_CHANGER_QUEUE_FULL', 0, 'retry later') Thread media_thread 00:01:39 stkenmvr147a.fnal.gov 009702 root I LTO4_47MV Media Changer returned MEDIA_CHANGER_QUEUE_FULL Thread media_thread 00:01:39 stkensrv0n.fnal.gov 028796 enstore I FILSRV Updating update file set deleted = 'n', pnfs_path = '/pnfs/fnal.gov/usr/exp-db/monthly/cdf-offline/cdfofpr2/2009/11-November/01/channel01/CDFOFPR2_t701801352_s30277_p5', drive = 'stkenmvr202a:/dev/rmt/tps0d0n:1310051194', pnfs_id = '00300000000000000619A558' where bfid = 'CDMS125714169900000'; 00:01:40 stkensrv4n.fnal.gov 003689 enstore I SL8500G1MC MC Overflow: 4 mount VOR381 0,5,1,3 00:01:40 stkenmvr149a.fnal.gov 006318 root E LTO4_49MV loadvol ('MEDIA_CHANGER_QUEUE_FULL', 0, 'retry later') Thread media_thread 00:01:40 stkenmvr149a.fnal.gov 006318 root I LTO4_49MV Media Changer returned MEDIA_CHANGER_QUEUE_FULL Thread media_thread 00:01:40 stkendca20a.fnal.gov 032240 root I ENCP MSG_TYPE=ENCP_XFER write_to_hsm /diskb/write-pool-2/data/00300000000000000619A558 -> /pnfs/fnal.gov/usr/exp-db/monthly/cdf-offline/cdfofpr2/2009/11-November/01/channel01/CDFOFPR2_t701801352_s30277_p5: 2147475456 bytes copied to VON218 at 31 MB/S (59.3 MB/S network) (87.1 MB/S drive) (33 MB/S disk) mover=LTO4_102.mover drive_id=ULTRIUM-TD4 drive_sn=1310051194 drive_vendor=IBM elapsed=659.02 {'storage_group': 'exp-db', 'encp_crc': 1811615136L, 'disk_rate': 32.960308354846418, 'drive_rate': 87.12195610763942, 'driver': 'FTTDriver', 'transfer_rate': 31.026755760875641, 'mover_interface': 'stkenmvr202a', 'overall_rate': 30.770105059115512, 'media_changer': 'SL8500G1.media_changer', 'network_rate': 31.026755760875641, 'encp_ip': '131.225.13.12', 'unique_id': 'stkendca20a.fnal.gov-1257141043-32240-0'} 00:01:40 stkendca20a.fnal.gov 032240 root I ENCP File /pnfs/fnal.gov/usr/exp-db/monthly/cdf-offline/cdfofpr2/2009/11-November/01/channel01/CDFOFPR2_t701801352_s30277_p5 copied successfully. 00:01:40 stkendca20a.fnal.gov 032240 root I ENCP Completed transferring 2147475456 bytes in 1 files in 659.025172949 sec. Overall rate = 30.8 MB/sec. Transfer rate = 31 MB/sec. Network rate = 59.3 MB/sec. Drive rate = 87.1 MB/sec. Disk rate = 33 MB/sec. Exit status = 0. 00:01:40 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:01:40 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM write_to_hsm next work on vol=VON218 mover=LTO4_102.mover requester:root 00:01:40 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM HAVE_BOUND:sending write_to_hsm /pnfs/fnal.gov/usr/exp-db/monthly/cdf-offline/cdfofpr2/2009/11-November/01/channel01/CDFOFPR2_t701800988_s30274_p1 to mover LTO4_102.mover ('131.225.13.197', 7862) DEL_DISM None 00:01:40 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM MSG_TYPE=ADD_TO_LMQ request added to work at movers queue for storage group : exp-db 00:01:40 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM HAVE_BOUND:sending {'vc': {'storage_group': 'exp-db', 'external_label': 'VON218', 'library': 'CD-LTO4G1', 'file_family': 'monthly-cdf-offline', 'wrapper': 'cpio_odc', 'address': ('131.225.13.59', 7502), 'volume_family': 'exp-db.monthly-cdf-offline.cpio_odc', 'file_family_width': 2}, 'outfilepath': '/pnfs/fnal.gov/usr/exp-db/monthly/cdf-offline/cdfofpr2/2009/11-November/01/channel01/CDFOFPR2_t701800988_s30274_p1', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 1, 'delayed_dismount': None, 'curpri': 1}, 'file_size': 2147475456L, 'mover': 'LTO4_102.mover', 'ignore_fair_share': None, 'retry': None, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fnal.gov/usr/exp-db/monthly/cdf-offline/cdfofpr2/2009/11-November/01/channel01/CDFOFPR2_t701800988_s30274_p1', 'uid': 0, 'uname': 'root', 'type': 'cpio_odc', 'gid': 0, 'mtime': 1257141044, 'rmajor': 0, 'machine': ('Linux', 'stkendca20a.fnal.gov', '2.6.9-89.0.9.ELsmp', '#1 SMP Mon Aug 24 13:47:43 CDT 2009', 'i686'), 'sanity_size': 65536, 'pstat': (16877, 820663368L, 20L, 1, 1602, 2752, 512L, 1257130346, 1257130346, 1254398469), 'mode_octal': '0100016', 'mode': 33188, 'gname': 'root', 'size_bytes': 2147475456L, 'fullname': '/diskb/write-pool-2/data/003000000000000006199FF8', 'inode': 806985720L, 'minor': 20}, 'version': 'v3_8b CVS $Revision: 1.922 $ ', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/diskb/write-pool-2/data/003000000000000006199FF8', 'status': ('ok', None), 'outfile': '/pnfs/fnal.gov/usr/exp-db/monthly/cdf-offline/cdfofpr2/2009/11-November/01/channel01/.(access)(003000000000000006199FF8)', 'fc': {'size': 2147475456L, 'external_label': 'VON218', 'address': ('131.225.13.59', 7501)}, 'infilepath': '/diskb/write-pool-2/data/003000000000000006199FF8', 'at_the_top': 258, 'local_inode': 271274707L, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.13.12', 57127), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'lm_dequeued': 1257141700.667254, 'encp_start_time': 1257141042.5807979, 'job_queued': 1257141044.964268, 't0': 1257141042, 'in_queue': 655.70270991325378}, 'user_level': 'user2', 'unique_id': 'stkendca20a.fnal.gov-1257141044-32276-0'} to mover 00:01:40 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:01:40 stkenmvr202a.fnal.gov 006121 root I LTO4_102MV WRITE_TO_HSM Thread MainThread 00:01:40 stkenmvr202a.fnal.gov 006121 root I LTO4_102MV SENDING {'vc': {'storage_group': 'exp-db', 'external_label': 'VON218', 'library': 'CD-LTO4G1', 'file_family': 'monthly-cdf-offline', 'wrapper': 'cpio_odc', 'address': ('131.225.13.59', 7502), 'volume_family': 'exp-db.monthly-cdf-offline.cpio_odc', 'file_family_width': 2}, 'outfilepath': '/pnfs/fnal.gov/usr/exp-db/monthly/cdf-offline/cdfofpr2/2009/11-November/01/channel01/CDFOFPR2_t701800988_s30274_p1', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 1, 'delayed_dismount': None, 'curpri': 1}, 'file_size': 2147475456L, 'mover': {'max_buffer': 2621440000L, 'check_written_file': 600, 'max_failures': 5, 'library': 'CD-LTO4G1.library_manager', 'local_mover': 0, 'single_filemark': 1, 'max_consecutive_failures': 3, 'norestart': 'INQ', 'port': 7862, 'mc_device': '0,6,1,14', 'compression': 0, 'serial_num': '1310051194', 'update_interval_in_bound': 2, 'update_interval': 5, 'syslog_entry': 'sense|st[0-9]', 'status': ('ok', None), 'max_rate': 73400320.0, 'vendor_id': 'IBM', 'driver': 'FTTDriver', 'host': 'stkenmvr202a', 'max_time_in_state': 1200, 'device': 'stkenmvr202a:/dev/rmt/tps0d0n', 'name': 'LTO4_102.mover', 'send_stats': 1, 'product_id': 'ULTRIUM-TD4', 'statistics_path': '/tmp/enstore/enstore/LTO4_102.stat', 'mover_address': ('131.225.13.197', 7862), 'mount_delay': 15, 'check_first_written_file': 0, 'callback_addr': ('131.225.13.197', 57458), 'data_ip': 'stkenmvr202a', 'logname': 'LTO4_102MV', 'hostip': '131.225.13.197', 'do_cleaning': 'No', 'media_changer': 'SL8500G1.media_changer'}, 'ignore_fair_share': None, 'retry': None, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fnal.gov/usr/exp-db/monthly/cdf-offline/cdfofpr2/2009/11-November/01/channel01/CDFOFPR2_t701800988_s30274_p1', 'uid': 0, 'uname': 'root', 'mtime': 1257141044, 'rmajor': 0, 'machine': ('Linux', 'stkendca20a.fnal.gov', '2.6.9-89.0.9.ELsmp', '#1 SMP Mon Aug 24 13:47:43 CDT 2009', 'i686'), 'sanity_size': 65536, 'type': 'cpio_odc', 'pstat': (16877, 820663368L, 20L, 1, 1602, 2752, 512L, 1257130346, 1257130346, 1254398469), 'mode_octal': '0100016', 'mode': 33188, 'size_bytes': 2147475456L, 'gname': 'root', 'gid': 0, 'fullname': '/diskb/write-pool-2/data/003000000000000006199FF8', 'inode': 806985720L, 'minor': 20}, 'version': 'v3_8b CVS $Revision: 1.922 $ ', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/diskb/write-pool-2/data/003000000000000006199FF8', 'status': ('ok', None), 'outfile': '/pnfs/fnal.gov/usr/exp-db/monthly/cdf-offline/cdfofpr2/2009/11-November/01/channel01/.(access)(003000000000000006199FF8)', 'fc': {'size': 2147475456L, 'external_label': 'VON218', 'address': ('131.225.13.59', 7501)}, 'infilepath': '/diskb/write-pool-2/data/003000000000000006199FF8', 'at_the_top': 258, 'local_inode': 271274707L, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.13.12', 57127), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'lm_dequeued': 1257141700.667254, 'encp_start_time': 1257141042.5807979, 'job_queued': 1257141044.964268, 't0': 1257141042, 'in_queue': 655.70270991325378}, 'user_level': 'user2', 'unique_id': 'stkendca20a.fnal.gov-1257141044-32276-0'} Thread client_connect_thread 00:01:40 stkendca20a.fnal.gov 032276 root I ENCP Control socket ('131.225.13.12', 57127) is connected to ('131.225.13.197', 57459) for stkendca20a.fnal.gov-1257141044-32276-0. elapsed=658.120sec 00:01:40 stkendca20a.fnal.gov 032276 root I ENCP Received callback ticket from mover LTO4_102.mover for transfer stkendca20a.fnal.gov-1257141044-32276-0. elapsed=658.120sec 00:01:40 stkendca20a.fnal.gov 032276 root I ENCP Atempting to connect data socket to mover at ('131.225.13.197', 57458). 00:01:40 stkendca20a.fnal.gov 032276 root I ENCP Connecting data socket to mover (('131.225.13.197', 57458)) with interface 131.225.13.12. elapsed=658.120sec 00:01:40 stkendca20a.fnal.gov 032276 root I ENCP Data socket ('131.225.13.12', 57177) is connected to ('131.225.13.197', 57458). elapsed=658.120sec 00:01:40 stkendca20a.fnal.gov 032276 root I ENCP Opening local file. 00:01:40 stkenmvr202a.fnal.gov 006121 root I LTO4_102MV tape position: target 332 current 332 0 Thread seek_thread 00:01:40 stkenmvr202a.fnal.gov 006121 root I LTO4_102MV write_tape starting, bytes_to_write=2147475968 Thread tape_thread 00:01:40 stkenmvr202a.fnal.gov 006121 root I LTO4_102MV Write starting. Tape VON218 absolute location in blocks 4954384 Thread tape_thread 00:01:40 stkendca20a.fnal.gov 032276 root I ENCP pf_status = ('ok', None) skip_layer_cleanup = False 00:01:41 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:01:42 last message repeated 1 times 00:01:42 stkendca20a.fnal.gov 032249 root I ENCP File /diskb/write-pool-2/data/00300000000000000619AFC8 transfered for stkendca20a.fnal.gov-1257141044-32249-0 in 59.8669128418 seconds. 00:01:42 stkensrv4n.fnal.gov 030308 enstore I SL8500G1MC query drive 0,6,1,13 => 0,0,6,1,13 online available IBM-LTO4 00:01:42 stkensrv4n.fnal.gov 030308 enstore I SL8500G1MC Dismount 14 ignored: Drive 0,6,1,13 is empty. Thought VOR382 was there =>. dismount VOLUME 0,6,1,13 force => ok,0,6,1,13 online available IBM-LTO4 00:01:42 stkensrv4n.fnal.gov 030311 enstore I SL8500G1MC query drive 0,5,1,15 => 0,0,5,1,15 online available IBM-LTO4 00:01:42 stkensrv4n.fnal.gov 030311 enstore I SL8500G1MC Dismount 14 ignored: Drive 0,5,1,15 is empty. Thought VON584 was there =>. dismount VOLUME 0,5,1,15 force => ok,0,5,1,15 online available IBM-LTO4 00:01:42 stkensrv4n.fnal.gov 003689 enstore I SL8500G1MC MC Overflow: 4 mount VOP202 0,5,1,12 00:01:42 stkenmvr151a.fnal.gov 013228 root E LTO4_51MV loadvol ('MEDIA_CHANGER_QUEUE_FULL', 0, 'retry later') Thread media_thread 00:01:42 stkenmvr151a.fnal.gov 013228 root I LTO4_51MV Media Changer returned MEDIA_CHANGER_QUEUE_FULL Thread media_thread 00:01:42 stkenmvr208a.fnal.gov 006111 root I LTO4_108MV written bytes 2147475968/2147475968, blocks 16384 header 190 trailer 322 Thread tape_thread 00:01:43 stkenmvr203a.fnal.gov 006116 root I LTO4_103MV written bytes 2099840000/2099840000, blocks 16021 header 292 trailer 533 Thread tape_thread 00:01:43 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:01:44 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:01:44 stkenmvr143a.fnal.gov 028425 root I LTO4_43MV skip_fm 89 Thread seek_thread 00:01:44 stkenmvr143a.fnal.gov 028425 root I LTO4_43MV write_tape starting, bytes_to_write=2107765760 Thread tape_thread 00:01:44 stkenmvr143a.fnal.gov 028425 root I LTO4_43MV Write starting. Tape VOR416 absolute location in blocks 2089544 Thread tape_thread 00:01:45 stkenmvr203a.fnal.gov 006116 root I LTO4_103MV filemarks written. Tape VOR423 absolute location in blocks 930312 Thread tape_thread 00:01:45 stkenmvr203a.fnal.gov 006116 root I LTO4_103MV write_tape timing:pull 0.21901679039 write 6.43847513199 crc_check 9.51273107529 freespace 0.158297538757 block_write 16.3285205364 idle 0.287342309952 Thread tape_thread 00:01:45 stkenmvr203a.fnal.gov 006116 root I LTO4_103MV drive stats after write. Tape VOR423 position 38 block 0 block_size 0 bloc_loc 930312 tot_blocks 762939000 BOT 0 read_err 0 write_err 0 bytes 2099840000 block_write_tot 16.3285205364 tape_rate 128599525.923 Thread tape_thread 00:01:45 stkenmvr203a.fnal.gov 006116 root I LTO4_103MV remaining bytes info in DB 663811072000 estimated 661711232000 reported from drive 661842944000 position 39 Thread tape_thread 00:01:45 stkenmvr203a.fnal.gov 006116 root I LTO4_103MV new bitfile request {'uid': 0, 'external_label': 'VOR423', 'gid': 0, 'location_cookie': '0000_000000000_0000038', 'complete_crc': 3369225047L, 'sanity_cookie': (65536L, 801187488L), 'size': 2099839175L} Thread tape_thread 00:01:45 stkenmvr203a.fnal.gov 006116 root I LTO4_103MV set remaining: VOR423 661842944000 0000_000000000_0000039 Thread tape_thread 00:01:45 last message repeated 1 times 00:01:45 stkensrv0n.fnal.gov 028762 enstore I VOLSRV Updating update volume set last_access = '2009-11-02 00:01:45', eod_cookie = '0000_000000000_0000039', non_del_files = '38', remaining_bytes = 661842944000 where label = 'VOR423'; 00:01:45 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:01:45 stkenmvr203a.fnal.gov 006116 root I LTO4_103MV transfer complete volume=VOR423 location=39 Thread tape_thread 00:01:45 last message repeated 1 times 00:01:45 stkensrv0n.fnal.gov 028762 enstore I VOLSRV Updating update volume set sum_wr_access = '38', modification_time = '2009-11-02 00:01:45' where label = 'VOR423'; 00:01:45 stkenmvr203a.fnal.gov 006116 root I LTO4_103MV Sending done to client: {'vc': {'comment': '', 'storage_group': 'cms', 'blocksize': 131072, 'declared': 1255358929.0, 'library': 'CD-LTO4G1', 'si_time': [0.0, 1246283775.0], 'user_inhibit': ['none', 'none'], 'system_inhibit': ['none', 'none'], 'external_label': 'VOR423', 'wrapper': 'cpio_odc', 'remaining_bytes': 663811072000L, 'sum_mounts': 79, 'sum_rd_access': 0, 'file_family_width': 15, 'last_access': 1257141663.0, 'media_type': 'LTO4', 'status': ('ok', None), 'eod_cookie': '0000_000000000_0000038', 'capacity_bytes': 858993459200L, 'non_del_files': 37, 'sum_wr_err': 0, 'sum_wr_access': 37, 'file_family': 'BACKFILL2', 'address': ('131.225.13.59', 7502), 'volume_family': 'cms.BACKFILL2.cpio_odc', 'modification_time': 1257141663.0, 'write_protected': 'n', 'sum_rd_err': 0, 'first_access': 1257122479.0}, 'outfilepath': '/pnfs/fnal.gov/usr/cms/WAX//11/store/backfill/2/data/IansMagicMushroomSoup-T0Test-AnalyzeThisAndGetAFreeBigFatHairyPhD/JetMonitor/RAW/v10_31_pre20replaythingy_v15/000/000/015/80F0E71A-45C6-DE11-A8B3-001D09F29597.root', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 25, 'delayed_dismount': None, 'curpri': 25}, 'file_size': 2099839175L, 'mover': {'max_buffer': 2621440000L, 'check_written_file': 600, 'max_failures': 5, 'library': 'CD-LTO4G1.library_manager', 'local_mover': 0, 'single_filemark': 1, 'max_consecutive_failures': 3, 'norestart': 'INQ', 'port': 7863, 'mc_device': '0,6,1,2', 'compression': 0, 'serial_num': '1310050821', 'update_interval_in_bound': 2, 'update_interval': 5, 'syslog_entry': 'sense|st[0-9]', 'status': ('ok', None), 'max_rate': 73400320.0, 'vendor_id': 'IBM', 'driver': 'FTTDriver', 'host': 'stkenmvr203a', 'max_time_in_state': 1200, 'device': 'stkenmvr203a:/dev/rmt/tps0d0n', 'name': 'LTO4_103.mover', 'send_stats': 1, 'product_id': 'ULTRIUM-TD4', 'statistics_path': '/tmp/enstore/enstore/LTO4_103.stat', 'mover_address': ('131.225.13.198', 7863), 'mount_delay': 15, 'check_first_written_file': 0, 'callback_addr': ('131.225.13.198', 57831), 'data_ip': 'stkenmvr203a', 'logname': 'LTO4_103MV', 'hostip': '131.225.13.198', 'do_cleaning': 'No', 'media_changer': 'SL8500G1.media_changer'}, 'ignore_fair_share': None, 'retry': None, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fnal.gov/usr/cms/WAX//11/store/backfill/2/data/IansMagicMushroomSoup-T0Test-AnalyzeThisAndGetAFreeBigFatHairyPhD/JetMonitor/RAW/v10_31_pre20replaythingy_v15/000/000/015/80F0E71A-45C6-DE11-A8B3-001D09F29597.root', 'uid': 0, 'uname': 'root', 'mtime': 1257141677, 'rmajor': 0, 'machine': ('Linux', 'cmsstor194.fnal.gov', '2.6.31', '#1 SMP Mon Sep 14 09:43:20 CDT 2009', 'x86_64'), 'sanity_size': 65536, 'type': 'cpio_odc', 'pstat': (16877, 126372832, 21L, 1, 9811, 5063, 512, 1257141604, 1257141604, 1257107163), 'mode_octal': '0100016', 'mode': 33188, 'size_bytes': 2099839175L, 'gname': 'root', 'gid': 0, 'fullname': '/storage/data1/write-pool-1/data/0007000000000000618C8FA8', 'inode': 126652328L, 'minor': 21}, 'version': 'v3_7d CVS $Revision: 1.922 $ encp', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/storage/data1/write-pool-1/data/0007000000000000618C8FA8', 'status': ('ok', None), 'outfile': '/pnfs/fnal.gov/usr/cms/WAX//11/store/backfill/2/data/IansMagicMushroomSoup-T0Test-AnalyzeThisAndGetAFreeBigFatHairyPhD/JetMonitor/RAW/v10_31_pre20replaythingy_v15/000/000/015/.(access)(0007000000000000618C8FA8)', 'fc': {'uid': 0, 'external_label': 'VOR423', 'gid': 0, 'location_cookie': '0000_000000000_0000038', 'complete_crc': 3369225047L, 'bfid': 'CDMS125714170500000', 'sanity_cookie': (65536L, 801187488L), 'size': 2099839175L}, 'infilepath': '/storage/data1/write-pool-1/data/0007000000000000618C8FA8', 'at_the_top': 5, 'local_inode': 7339, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.207.19', 41677), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'encp_start_time': 1257141675.1245601, 'seek_time': 0.029788017272949219, 'transfer_time': 26.743173837661743, 't0': 1257141675(trunc. 4379) Thread tape_thread 00:01:45 stkenmvr203a.fnal.gov 006116 root I LTO4_103MV Done is sent Thread tape_thread 00:01:45 stkenmvr203a.fnal.gov 006116 root I LTO4_103MV maybe_clean Thread tape_thread 00:01:45 cmsstor194.fnal.gov 016956 root I ENCP Received final dialog for cmsstor194.fnal.gov-1257141677-16956-0. elapsed=30.370sec 00:01:45 cmsstor194.fnal.gov 016956 root I ENCP leaving transfer_file(). 00:01:45 stkenmvr203a.fnal.gov 006116 root I LTO4_103MV returned from maybe_clean Thread tape_thread 00:01:45 cmsstor194.fnal.gov 016956 root I ENCP pf_status = ('ok', None) skip_layer_cleanup = False 00:01:45 stkensrv4n.fnal.gov 030311 enstore I SL8500G1MC query vol VON584 => 0,VON584 home 0, 9,16, 3, 0 LTO-800G 00:01:45 stkensrv4n.fnal.gov 030308 enstore I SL8500G1MC query vol VOR382 => 0,VOR382 home 0, 9,15, 5, 1 LTO-800G 00:01:45 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:01:45 stkensrv0n.fnal.gov 028796 enstore I FILSRV Updating update file set deleted = 'n', pnfs_path = '/pnfs/fnal.gov/usr/cms/WAX//11/store/backfill/2/data/IansMagicMushroomSoup-T0Test-AnalyzeThisAndGetAFreeBigFatHairyPhD/JetMonitor/RAW/v10_31_pre20replaythingy_v15/000/000/015/80F0E71A-45C6-DE11-A8B3-001D09F29597.root', drive = 'stkenmvr203a:/dev/rmt/tps0d0n:1310050821', pnfs_id = '0007000000000000618C8FA8' where bfid = 'CDMS125714170500000'; 00:01:45 cmsstor194.fnal.gov 016956 root I ENCP MSG_TYPE=ENCP_XFER write_to_hsm /storage/data1/write-pool-1/data/0007000000000000618C8FA8 -> /pnfs/fnal.gov/usr/cms/WAX//11/store/backfill/2/data/IansMagicMushroomSoup-T0Test-AnalyzeThisAndGetAFreeBigFatHairyPhD/JetMonitor/RAW/v10_31_pre20replaythingy_v15/000/000/015/80F0E71A-45C6-DE11-A8B3-001D09F29597.root: 2099839175 bytes copied to VOR423 at 75.4 MB/S (97.5 MB/S network) (109 MB/S drive) (113 MB/S disk) mover=LTO4_103.mover drive_id=ULTRIUM-TD4 drive_sn=1310050821 drive_vendor=IBM elapsed=30.702 {'storage_group': 'cms', 'encp_crc': 3369225047L, 'disk_rate': 112.79341997659465, 'drive_rate': 108.92958063282921, 'driver': 'FTTDriver', 'transfer_rate': 75.350560795541824, 'mover_interface': 'stkenmvr203a', 'overall_rate': 73.755609807156461, 'media_changer': 'SL8500G1.media_changer', 'network_rate': 75.350560795541824, 'encp_ip': '131.225.207.19', 'unique_id': 'cmsstor194.fnal.gov-1257141677-16956-0'} 00:01:45 cmsstor194.fnal.gov 016956 root I ENCP File /pnfs/fnal.gov/usr/cms/WAX//11/store/backfill/2/data/IansMagicMushroomSoup-T0Test-AnalyzeThisAndGetAFreeBigFatHairyPhD/JetMonitor/RAW/v10_31_pre20replaythingy_v15/000/000/015/80F0E71A-45C6-DE11-A8B3-001D09F29597.root copied successfully. 00:01:45 cmsstor194.fnal.gov 016956 root I ENCP Completed transferring 2099839175 bytes in 1 files in 30.7028319836 sec. Overall rate = 73.8 MB/sec. Transfer rate = 75.4 MB/sec. Network rate = 97.5 MB/sec. Drive rate = 109 MB/sec. Disk rate = 113 MB/sec. Exit status = 0. 00:01:45 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:01:45 stkenmvr196a.fnal.gov 027637 root I LTO4_96MV Tape is ejected Thread media_thread 00:01:45 stkenmvr196a.fnal.gov 027637 root I LTO4_96MV dismounting VON705 Thread media_thread 00:01:45 stkensrv4n.fnal.gov 003689 enstore I SL8500G1MC MC Overflow: 4 dismount VON705 0,7,1,2 00:01:45 stkenmvr196a.fnal.gov 027637 root E LTO4_96MV unloadvol ('MEDIA_CHANGER_QUEUE_FULL', 0, 'retry later') Thread media_thread 00:01:45 stkenmvr196a.fnal.gov 027637 root I LTO4_96MV Media Changer returned MEDIA_CHANGER_QUEUE_FULL Thread media_thread 00:01:45 stkensrv4n.fnal.gov 003689 enstore I SL8500G1MC MC Overflow: 4 mount VOR422 0,6,1,8 00:01:45 stkenmvr206a.fnal.gov 026766 root E LTO4_106MV loadvol ('MEDIA_CHANGER_QUEUE_FULL', 0, 'retry later') Thread media_thread 00:01:45 stkenmvr206a.fnal.gov 026766 root I LTO4_106MV Media Changer returned MEDIA_CHANGER_QUEUE_FULL Thread media_thread 00:01:46 stkenmvr208a.fnal.gov 006111 root I LTO4_108MV filemarks written. Tape VON217 absolute location in blocks 4955228 Thread tape_thread 00:01:46 stkenmvr208a.fnal.gov 006111 root I LTO4_108MV write_tape timing:pull 0.221731901169 write 10.0576488972 crc_check 9.88184833527 freespace 0.165273666382 block_write 20.3265028 idle 0.293092012405 Thread tape_thread 00:01:46 stkenmvr208a.fnal.gov 006111 root I LTO4_108MV drive stats after write. Tape VON217 position 324 block 0 block_size 0 bloc_loc 4955228 tot_blocks 762939000 BOT 0 read_err 0 write_err 0 bytes 2147475968 block_write_tot 20.3265028 tape_rate 105649062.661 Thread tape_thread 00:01:46 stkenmvr208a.fnal.gov 006111 root I LTO4_108MV remaining bytes info in DB 168145920000 estimated 165998444032 reported from drive 166111232000 position 325 Thread tape_thread 00:01:46 stkenmvr208a.fnal.gov 006111 root I LTO4_108MV new bitfile request {'uid': 0, 'external_label': 'VON217', 'gid': 0, 'location_cookie': '0000_000000000_0000324', 'complete_crc': 1062925331L, 'sanity_cookie': (65536L, 3272666131L), 'size': 2147475456L} Thread tape_thread 00:01:46 stkensrv2n.fnal.gov 006421 enstore E INQSRV Could not create outage file lock file (/srv2/enstore/www/web-pages//enstore_outage.py.lock) 00:01:46 stkenmvr208a.fnal.gov 006111 root I LTO4_108MV set remaining: VON217 166111232000 0000_000000000_0000325 Thread tape_thread 00:01:46 last message repeated 1 times 00:01:46 stkensrv0n.fnal.gov 028762 enstore I VOLSRV Updating update volume set last_access = '2009-11-02 00:01:46', eod_cookie = '0000_000000000_0000325', non_del_files = '354', remaining_bytes = 166111232000 where label = 'VON217'; 00:01:46 stkensrv2n.fnal.gov 006421 enstore E INQSRV Could not create outage file lock file (/srv2/enstore/www/web-pages//enstore_outage.py.lock) 00:01:46 stkenmvr208a.fnal.gov 006111 root I LTO4_108MV transfer complete volume=VON217 location=325 Thread tape_thread 00:01:46 last message repeated 1 times 00:01:46 stkensrv0n.fnal.gov 028762 enstore I VOLSRV Updating update volume set sum_wr_access = '324', modification_time = '2009-11-02 00:01:46' where label = 'VON217'; 00:01:46 stkenmvr208a.fnal.gov 006111 root I LTO4_108MV Sending done to client: {'vc': {'comment': '', 'storage_group': 'exp-db', 'blocksize': 131072, 'declared': 1240352603.0, 'library': 'CD-LTO4G1', 'si_time': [0.0, 0.0], 'user_inhibit': ['none', 'none'], 'system_inhibit': ['none', 'none'], 'external_label': 'VON217', 'wrapper': 'cpio_odc', 'remaining_bytes': 168145920000L, 'sum_mounts': 62, 'sum_rd_access': 0, 'file_family_width': 2, 'last_access': 1257141641.0, 'media_type': 'LTO4', 'status': ('ok', None), 'eod_cookie': '0000_000000000_0000324', 'capacity_bytes': 858993459200L, 'non_del_files': 353, 'sum_wr_err': 0, 'sum_wr_access': 323, 'file_family': 'monthly-cdf-offline', 'address': ('131.225.13.59', 7502), 'volume_family': 'exp-db.monthly-cdf-offline.cpio_odc', 'modification_time': 1257141641.0, 'write_protected': 'n', 'sum_rd_err': 0, 'first_access': 1241268501.0}, 'outfilepath': '/pnfs/fnal.gov/usr/exp-db/monthly/cdf-offline/cdfofpr2/2009/11-November/01/channel01/CDFOFPR2_t701802222_s30285_p1', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 1, 'delayed_dismount': None, 'curpri': 1}, 'file_size': 2147475456L, 'mover': {'max_buffer': 2621440000L, 'check_written_file': 600, 'max_failures': 5, 'library': 'CD-LTO4G1.library_manager', 'local_mover': 0, 'single_filemark': 1, 'max_consecutive_failures': 3, 'norestart': 'INQ', 'port': 7868, 'mc_device': '0,5,1,11', 'compression': 0, 'serial_num': '1310154353', 'update_interval_in_bound': 2, 'update_interval': 5, 'syslog_entry': 'sense|st[0-9]', 'status': ('ok', None), 'max_rate': 73400320.0, 'vendor_id': 'IBM', 'driver': 'FTTDriver', 'host': 'stkenmvr208a', 'max_time_in_state': 1200, 'device': 'stkenmvr208a:/dev/rmt/tps0d0n', 'name': 'LTO4_108.mover', 'send_stats': 1, 'product_id': 'ULTRIUM-TD4', 'statistics_path': '/tmp/enstore/enstore/LTO4_108.stat', 'mover_address': ('131.225.13.204', 7868), 'mount_delay': 15, 'check_first_written_file': 0, 'callback_addr': ('131.225.13.204', 57957), 'data_ip': 'stkenmvr208a', 'logname': 'LTO4_108MV', 'hostip': '131.225.13.204', 'do_cleaning': 'No', 'media_changer': 'SL8500G1.media_changer'}, 'ignore_fair_share': None, 'retry': None, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fnal.gov/usr/exp-db/monthly/cdf-offline/cdfofpr2/2009/11-November/01/channel01/CDFOFPR2_t701802222_s30285_p1', 'uid': 0, 'uname': 'root', 'mtime': 1257141044, 'rmajor': 0, 'machine': ('Linux', 'stkendca20a.fnal.gov', '2.6.9-89.0.9.ELsmp', '#1 SMP Mon Aug 24 13:47:43 CDT 2009', 'i686'), 'sanity_size': 65536, 'type': 'cpio_odc', 'pstat': (16877, 820663368L, 20L, 1, 1602, 2752, 512L, 1257130346, 1257130346, 1254398469), 'mode_octal': '0100016', 'mode': 33188, 'size_bytes': 2147475456L, 'gname': 'root', 'gid': 0, 'fullname': '/diskb/write-pool-2/data/00300000000000000619AFC8', 'inode': 806989768L, 'minor': 20}, 'version': 'v3_8b CVS $Revision: 1.922 $ ', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/diskb/write-pool-2/data/00300000000000000619AFC8', 'status': ('ok', None), 'outfile': '/pnfs/fnal.gov/usr/exp-db/monthly/cdf-offline/cdfofpr2/2009/11-November/01/channel01/.(access)(00300000000000000619AFC8)', 'fc': {'uid': 0, 'external_label': 'VON217', 'gid': 0, 'location_cookie': '0000_000000000_0000324', 'complete_crc': 1062925331L, 'bfid': 'CDMS125714170600000', 'sanity_cookie': (65536L, 3272666131L), 'size': 2147475456L}, 'infilepath': '/diskb/write-pool-2/data/00300000000000000619AFC8', 'at_the_top': 52, 'local_inode': 271282418L, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.13.12', 57126), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'encp_start_time': 1257141041.831778, 'seek_time': 0.029545068740844727, 'transfer_time': 64.070245981216431, 't0': 1257141041, 'mount_time': 0.016988992691040039, 'lm_dequeued': 1257141642.189368, 'job_queued': 1257141044.8637171, 'drive_call_time': 9.8381516933441162, 'in_queue': 597.32538795471191, 'drive_transfer_time': 23.975114822387695}, 'user_level': 'user2', 'unique_id': 'stkendca20a.fnal.gov(trunc. 4101) Thread tape_thread 00:01:46 stkenmvr208a.fnal.gov 006111 root I LTO4_108MV Done is sent Thread tape_thread 00:01:46 stkenmvr208a.fnal.gov 006111 root I LTO4_108MV maybe_clean Thread tape_thread 00:01:46 stkenmvr208a.fnal.gov 006111 root I LTO4_108MV returned from maybe_clean Thread tape_thread 00:01:46 stkendca20a.fnal.gov 032249 root I ENCP Received final dialog for stkendca20a.fnal.gov-1257141044-32249-0. elapsed=665.220sec 00:01:46 stkendca20a.fnal.gov 032249 root I ENCP leaving transfer_file(). 00:01:46 stkensrv2n.fnal.gov 006421 enstore E INQSRV Could not read file /srv2/enstore/www/web-pages//enstore_outage.py 00:01:46 stkendca20a.fnal.gov 032249 root I ENCP pf_status = ('ok', None) skip_layer_cleanup = False 00:01:46 stkensrv0n.fnal.gov 028796 enstore I FILSRV Updating update file set deleted = 'n', pnfs_path = '/pnfs/fnal.gov/usr/exp-db/monthly/cdf-offline/cdfofpr2/2009/11-November/01/channel01/CDFOFPR2_t701802222_s30285_p1', drive = 'stkenmvr208a:/dev/rmt/tps0d0n:1310154353', pnfs_id = '00300000000000000619AFC8' where bfid = 'CDMS125714170600000'; 00:01:46 stkendca20a.fnal.gov 032249 root I ENCP MSG_TYPE=ENCP_XFER write_to_hsm /diskb/write-pool-2/data/00300000000000000619AFC8 -> /pnfs/fnal.gov/usr/exp-db/monthly/cdf-offline/cdfofpr2/2009/11-November/01/channel01/CDFOFPR2_t701802222_s30285_p1: 2147475456 bytes copied to VON217 at 32 MB/S (59.6 MB/S network) (85.4 MB/S drive) (34.2 MB/S disk) mover=LTO4_108.mover drive_id=ULTRIUM-TD4 drive_sn=1310154353 drive_vendor=IBM elapsed=665.14 {'storage_group': 'exp-db', 'encp_crc': 1062925331L, 'disk_rate': 34.21595555400507, 'drive_rate': 85.421579945369345, 'driver': 'FTTDriver', 'transfer_rate': 31.956151878797673, 'mover_interface': 'stkenmvr208a', 'overall_rate': 31.635963554202135, 'media_changer': 'SL8500G1.media_changer', 'network_rate': 31.956151878797673, 'encp_ip': '131.225.13.12', 'unique_id': 'stkendca20a.fnal.gov-1257141044-32249-0'} 00:01:46 stkendca20a.fnal.gov 032249 root I ENCP File /pnfs/fnal.gov/usr/exp-db/monthly/cdf-offline/cdfofpr2/2009/11-November/01/channel01/CDFOFPR2_t701802222_s30285_p1 copied successfully. 00:01:46 stkendca20a.fnal.gov 032249 root I ENCP Completed transferring 2147475456 bytes in 1 files in 665.153029919 sec. Overall rate = 31.6 MB/sec. Transfer rate = 32 MB/sec. Network rate = 59.6 MB/sec. Drive rate = 85.4 MB/sec. Disk rate = 34.2 MB/sec. Exit status = 0. 00:01:47 stkensrv2n.fnal.gov 006421 enstore E INQSRV Could not create outage file lock file (/srv2/enstore/www/web-pages//enstore_outage.py.lock) 00:01:47 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:01:47 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM write_to_hsm next work on vol=VON217 mover=LTO4_108.mover requester:root 00:01:47 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM HAVE_BOUND:sending write_to_hsm /pnfs/fnal.gov/usr/exp-db/monthly/cdf-offline/cdfofpr2/2009/11-November/01/channel01/CDFOFPR2_t701800872_s30272_p2 to mover LTO4_108.mover ('131.225.13.204', 7868) DEL_DISM None 00:01:47 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM MSG_TYPE=ADD_TO_LMQ request added to work at movers queue for storage group : exp-db 00:01:47 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM HAVE_BOUND:sending {'vc': {'storage_group': 'exp-db', 'external_label': 'VON217', 'library': 'CD-LTO4G1', 'file_family': 'monthly-cdf-offline', 'wrapper': 'cpio_odc', 'address': ('131.225.13.59', 7502), 'volume_family': 'exp-db.monthly-cdf-offline.cpio_odc', 'file_family_width': 2}, 'outfilepath': '/pnfs/fnal.gov/usr/exp-db/monthly/cdf-offline/cdfofpr2/2009/11-November/01/channel01/CDFOFPR2_t701800872_s30272_p2', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 1, 'delayed_dismount': None, 'curpri': 1}, 'file_size': 196173824L, 'mover': 'LTO4_108.mover', 'ignore_fair_share': None, 'retry': None, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fnal.gov/usr/exp-db/monthly/cdf-offline/cdfofpr2/2009/11-November/01/channel01/CDFOFPR2_t701800872_s30272_p2', 'uid': 0, 'uname': 'root', 'type': 'cpio_odc', 'gid': 0, 'mtime': 1257141531, 'rmajor': 0, 'machine': ('Linux', 'stkendca18a.fnal.gov', '2.6.9-89.0.9.ELsmp', '#1 SMP Mon Aug 24 13:47:43 CDT 2009', 'i686'), 'sanity_size': 65536, 'pstat': (16877, 820663368L, 20L, 1, 1602, 2752, 512L, 1257130346, 1257130346, 1254398469), 'mode_octal': '0100016', 'mode': 33188, 'gname': 'root', 'size_bytes': 196173824L, 'fullname': '/diskb/write-pool-2/data/003000000000000006199FC8', 'inode': 806985672L, 'minor': 20}, 'version': 'v3_8b CVS $Revision: 1.922 $ ', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/diskb/write-pool-2/data/003000000000000006199FC8', 'status': ('ok', None), 'outfile': '/pnfs/fnal.gov/usr/exp-db/monthly/cdf-offline/cdfofpr2/2009/11-November/01/channel01/.(access)(003000000000000006199FC8)', 'fc': {'size': 196173824L, 'external_label': 'VON217', 'address': ('131.225.13.59', 7501)}, 'infilepath': '/diskb/write-pool-2/data/003000000000000006199FC8', 'at_the_top': 38, 'local_inode': 569740957L, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.13.6', 43906), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'lm_dequeued': 1257141707.180501, 'encp_start_time': 1257141528.8155689, 'job_queued': 1257141531.5062211, 't0': 1257141528, 'in_queue': 175.67400884628296}, 'user_level': 'user2', 'unique_id': 'stkendca18a.fnal.gov-1257141531-16282-0'} to mover 00:01:47 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:01:47 stkenmvr208a.fnal.gov 006111 root I LTO4_108MV WRITE_TO_HSM Thread MainThread 00:01:47 stkenmvr208a.fnal.gov 006111 root I LTO4_108MV SENDING {'vc': {'storage_group': 'exp-db', 'external_label': 'VON217', 'library': 'CD-LTO4G1', 'file_family': 'monthly-cdf-offline', 'wrapper': 'cpio_odc', 'address': ('131.225.13.59', 7502), 'volume_family': 'exp-db.monthly-cdf-offline.cpio_odc', 'file_family_width': 2}, 'outfilepath': '/pnfs/fnal.gov/usr/exp-db/monthly/cdf-offline/cdfofpr2/2009/11-November/01/channel01/CDFOFPR2_t701800872_s30272_p2', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 1, 'delayed_dismount': None, 'curpri': 1}, 'file_size': 196173824L, 'mover': {'max_buffer': 2621440000L, 'check_written_file': 600, 'max_failures': 5, 'library': 'CD-LTO4G1.library_manager', 'local_mover': 0, 'single_filemark': 1, 'max_consecutive_failures': 3, 'norestart': 'INQ', 'port': 7868, 'mc_device': '0,5,1,11', 'compression': 0, 'serial_num': '1310154353', 'update_interval_in_bound': 2, 'update_interval': 5, 'syslog_entry': 'sense|st[0-9]', 'status': ('ok', None), 'max_rate': 73400320.0, 'vendor_id': 'IBM', 'driver': 'FTTDriver', 'host': 'stkenmvr208a', 'max_time_in_state': 1200, 'device': 'stkenmvr208a:/dev/rmt/tps0d0n', 'name': 'LTO4_108.mover', 'send_stats': 1, 'product_id': 'ULTRIUM-TD4', 'statistics_path': '/tmp/enstore/enstore/LTO4_108.stat', 'mover_address': ('131.225.13.204', 7868), 'mount_delay': 15, 'check_first_written_file': 0, 'callback_addr': ('131.225.13.204', 57960), 'data_ip': 'stkenmvr208a', 'logname': 'LTO4_108MV', 'hostip': '131.225.13.204', 'do_cleaning': 'No', 'media_changer': 'SL8500G1.media_changer'}, 'ignore_fair_share': None, 'retry': None, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fnal.gov/usr/exp-db/monthly/cdf-offline/cdfofpr2/2009/11-November/01/channel01/CDFOFPR2_t701800872_s30272_p2', 'uid': 0, 'uname': 'root', 'mtime': 1257141531, 'rmajor': 0, 'machine': ('Linux', 'stkendca18a.fnal.gov', '2.6.9-89.0.9.ELsmp', '#1 SMP Mon Aug 24 13:47:43 CDT 2009', 'i686'), 'sanity_size': 65536, 'type': 'cpio_odc', 'pstat': (16877, 820663368L, 20L, 1, 1602, 2752, 512L, 1257130346, 1257130346, 1254398469), 'mode_octal': '0100016', 'mode': 33188, 'size_bytes': 196173824L, 'gname': 'root', 'gid': 0, 'fullname': '/diskb/write-pool-2/data/003000000000000006199FC8', 'inode': 806985672L, 'minor': 20}, 'version': 'v3_8b CVS $Revision: 1.922 $ ', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/diskb/write-pool-2/data/003000000000000006199FC8', 'status': ('ok', None), 'outfile': '/pnfs/fnal.gov/usr/exp-db/monthly/cdf-offline/cdfofpr2/2009/11-November/01/channel01/.(access)(003000000000000006199FC8)', 'fc': {'size': 196173824L, 'external_label': 'VON217', 'address': ('131.225.13.59', 7501)}, 'infilepath': '/diskb/write-pool-2/data/003000000000000006199FC8', 'at_the_top': 38, 'local_inode': 569740957L, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.13.6', 43906), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'lm_dequeued': 1257141707.180501, 'encp_start_time': 1257141528.8155689, 'job_queued': 1257141531.5062211, 't0': 1257141528, 'in_queue': 175.67400884628296}, 'user_level': 'user2', 'unique_id': 'stkendca18a.fnal.gov-1257141531-16282-0'} Thread client_connect_thread 00:01:47 stkendca18a.fnal.gov 016282 root I ENCP Control socket ('131.225.13.6', 43906) is connected to ('131.225.13.204', 57961) for stkendca18a.fnal.gov-1257141531-16282-0. elapsed=178.400sec 00:01:47 stkendca18a.fnal.gov 016282 root I ENCP Received callback ticket from mover LTO4_108.mover for transfer stkendca18a.fnal.gov-1257141531-16282-0. elapsed=178.400sec 00:01:47 stkendca18a.fnal.gov 016282 root I ENCP Atempting to connect data socket to mover at ('131.225.13.204', 57960). 00:01:47 stkendca18a.fnal.gov 016282 root I ENCP Connecting data socket to mover (('131.225.13.204', 57960)) with interface 131.225.13.6. elapsed=178.400sec 00:01:47 stkendca18a.fnal.gov 016282 root I ENCP Data socket ('131.225.13.6', 43917) is connected to ('131.225.13.204', 57960). elapsed=178.400sec 00:01:47 stkendca18a.fnal.gov 016282 root I ENCP Opening local file. 00:01:47 stkenmvr208a.fnal.gov 006111 root I LTO4_108MV tape position: target 325 current 325 0 Thread seek_thread 00:01:47 stkendca18a.fnal.gov 016282 root I ENCP pf_status = ('ok', None) skip_layer_cleanup = False 00:01:47 stkenmvr208a.fnal.gov 006111 root I LTO4_108MV write_tape starting, bytes_to_write=196174336 Thread tape_thread 00:01:47 stkenmvr208a.fnal.gov 006111 root I LTO4_108MV Write starting. Tape VON217 absolute location in blocks 4955228 Thread tape_thread 00:01:47 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:01:47 stkenmvr198a.fnal.gov 015662 root I LTO4_98MV Tape is ejected Thread media_thread 00:01:47 stkenmvr198a.fnal.gov 015662 root I LTO4_98MV dismounting VOR201 Thread media_thread 00:01:47 stkensrv4n.fnal.gov 003689 enstore I SL8500G1MC MC Overflow: 4 dismount VOR201 0,7,1,1 00:01:47 stkenmvr198a.fnal.gov 015662 root E LTO4_98MV unloadvol ('MEDIA_CHANGER_QUEUE_FULL', 0, 'retry later') Thread media_thread 00:01:47 stkenmvr198a.fnal.gov 015662 root I LTO4_98MV Media Changer returned MEDIA_CHANGER_QUEUE_FULL Thread media_thread 00:01:47 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:01:48 last message repeated 1 times 00:01:48 stkensrv2n.fnal.gov 006421 enstore E INQSRV Could not create seen-down file lock file (/srv2/enstore/www/web-pages//enstore_seen_down.py.lock) 00:01:48 stkensrv2n.fnal.gov 006421 enstore E INQSRV Could not read file /srv2/enstore/www/web-pages//enstore_outage.py 00:01:48 stkensrv2n.fnal.gov 006421 enstore E INQSRV Could not read file /srv2/enstore/www/web-pages//enstore_seen_down.py 00:01:48 stkensrv2n.fnal.gov 006421 enstore E INQSRV Could not read file /srv2/enstore/www/web-pages//enstore_outage.py 00:01:48 stkensrv4n.fnal.gov 030308 enstore I SL8500G1MC query drive 0,6,1,13 => 0,0,6,1,13 online available IBM-LTO4 00:01:48 stkensrv4n.fnal.gov 030311 enstore I SL8500G1MC query drive 0,5,1,15 => 0,0,5,1,15 online available IBM-LTO4 00:01:49 stkendca18a.fnal.gov 016282 root I ENCP File /diskb/write-pool-2/data/003000000000000006199FC8 transfered for stkendca18a.fnal.gov-1257141531-16282-0 in 1.91507411003 seconds. 00:01:49 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:01:49 stkenmvr208a.fnal.gov 006111 root I LTO4_108MV written bytes 196174336/196174336, blocks 1497 header 190 trailer 322 Thread tape_thread 00:01:49 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:01:49 stkensrv4n.fnal.gov 003689 enstore I SL8500G1MC MC Overflow: 4 mount VOR203 0,6,1,0 00:01:49 stkenmvr147a.fnal.gov 009702 root E LTO4_47MV loadvol ('MEDIA_CHANGER_QUEUE_FULL', 0, 'retry later') Thread media_thread 00:01:49 stkenmvr147a.fnal.gov 009702 root I LTO4_47MV Media Changer returned MEDIA_CHANGER_QUEUE_FULL Thread media_thread 00:01:50 stkensrv4n.fnal.gov 003689 enstore I SL8500G1MC MC Overflow: 4 mount VOR381 0,5,1,3 00:01:50 stkenmvr149a.fnal.gov 006318 root E LTO4_49MV loadvol ('MEDIA_CHANGER_QUEUE_FULL', 0, 'retry later') Thread media_thread 00:01:50 stkenmvr149a.fnal.gov 006318 root I LTO4_49MV Media Changer returned MEDIA_CHANGER_QUEUE_FULL Thread media_thread 00:01:50 stkensrv2n.fnal.gov 006421 enstore E INQSRV Could not create seen-down file lock file (/srv2/enstore/www/web-pages//enstore_seen_down.py.lock) 00:01:50 stkensrv2n.fnal.gov 006421 enstore E INQSRV Could not read file /srv2/enstore/www/web-pages//enstore_seen_down.py 00:01:51 stkenmvr208a.fnal.gov 006111 root I LTO4_108MV filemarks written. Tape VON217 absolute location in blocks 4956726 Thread tape_thread 00:01:51 stkenmvr208a.fnal.gov 006111 root I LTO4_108MV write_tape timing:pull 0.0178024768829 write 0.707619667053 crc_check 0.92234826088 freespace 0.020450592041 block_write 1.66822099686 idle 0.0317823886871 Thread tape_thread 00:01:51 stkenmvr208a.fnal.gov 006111 root I LTO4_108MV drive stats after write. Tape VON217 position 325 block 0 block_size 0 bloc_loc 4956726 tot_blocks 762939000 BOT 0 read_err 0 write_err 0 bytes 196174336 block_write_tot 1.66822099686 tape_rate 117594932.787 Thread tape_thread 00:01:51 stkenmvr208a.fnal.gov 006111 root I LTO4_108MV remaining bytes info in DB 166111232000 estimated 165915057664 reported from drive 165924864000 position 326 Thread tape_thread 00:01:51 stkenmvr208a.fnal.gov 006111 root I LTO4_108MV new bitfile request {'uid': 0, 'external_label': 'VON217', 'gid': 0, 'location_cookie': '0000_000000000_0000325', 'complete_crc': 1690258267L, 'sanity_cookie': (65536L, 274209460L), 'size': 196173824L} Thread tape_thread 00:01:51 stkenmvr208a.fnal.gov 006111 root I LTO4_108MV set remaining: VON217 165924864000 0000_000000000_0000326 Thread tape_thread 00:01:51 last message repeated 1 times 00:01:51 stkensrv0n.fnal.gov 028762 enstore I VOLSRV Updating update volume set last_access = '2009-11-02 00:01:51', eod_cookie = '0000_000000000_0000326', non_del_files = '355', remaining_bytes = 165924864000 where label = 'VON217'; 00:01:51 stkenmvr208a.fnal.gov 006111 root I LTO4_108MV transfer complete volume=VON217 location=326 Thread tape_thread 00:01:51 last message repeated 1 times 00:01:51 stkensrv0n.fnal.gov 028762 enstore I VOLSRV Updating update volume set sum_wr_access = '325', modification_time = '2009-11-02 00:01:51' where label = 'VON217'; 00:01:51 stkenmvr208a.fnal.gov 006111 root I LTO4_108MV Sending done to client: {'vc': {'comment': '', 'storage_group': 'exp-db', 'blocksize': 131072, 'declared': 1240352603.0, 'library': 'CD-LTO4G1', 'si_time': [0.0, 0.0], 'user_inhibit': ['none', 'none'], 'system_inhibit': ['none', 'none'], 'external_label': 'VON217', 'wrapper': 'cpio_odc', 'remaining_bytes': 166111232000L, 'sum_mounts': 62, 'sum_rd_access': 0, 'file_family_width': 2, 'last_access': 1257141706.0, 'media_type': 'LTO4', 'status': ('ok', None), 'eod_cookie': '0000_000000000_0000325', 'capacity_bytes': 858993459200L, 'non_del_files': 354, 'sum_wr_err': 0, 'sum_wr_access': 324, 'file_family': 'monthly-cdf-offline', 'address': ('131.225.13.59', 7502), 'volume_family': 'exp-db.monthly-cdf-offline.cpio_odc', 'modification_time': 1257141706.0, 'write_protected': 'n', 'sum_rd_err': 0, 'first_access': 1241268501.0}, 'outfilepath': '/pnfs/fnal.gov/usr/exp-db/monthly/cdf-offline/cdfofpr2/2009/11-November/01/channel01/CDFOFPR2_t701800872_s30272_p2', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 1, 'delayed_dismount': None, 'curpri': 1}, 'file_size': 196173824L, 'mover': {'max_buffer': 2621440000L, 'check_written_file': 600, 'max_failures': 5, 'library': 'CD-LTO4G1.library_manager', 'local_mover': 0, 'single_filemark': 1, 'max_consecutive_failures': 3, 'norestart': 'INQ', 'port': 7868, 'mc_device': '0,5,1,11', 'compression': 0, 'serial_num': '1310154353', 'update_interval_in_bound': 2, 'update_interval': 5, 'syslog_entry': 'sense|st[0-9]', 'status': ('ok', None), 'max_rate': 73400320.0, 'vendor_id': 'IBM', 'driver': 'FTTDriver', 'host': 'stkenmvr208a', 'max_time_in_state': 1200, 'device': 'stkenmvr208a:/dev/rmt/tps0d0n', 'name': 'LTO4_108.mover', 'send_stats': 1, 'product_id': 'ULTRIUM-TD4', 'statistics_path': '/tmp/enstore/enstore/LTO4_108.stat', 'mover_address': ('131.225.13.204', 7868), 'mount_delay': 15, 'check_first_written_file': 0, 'callback_addr': ('131.225.13.204', 57960), 'data_ip': 'stkenmvr208a', 'logname': 'LTO4_108MV', 'hostip': '131.225.13.204', 'do_cleaning': 'No', 'media_changer': 'SL8500G1.media_changer'}, 'ignore_fair_share': None, 'retry': None, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fnal.gov/usr/exp-db/monthly/cdf-offline/cdfofpr2/2009/11-November/01/channel01/CDFOFPR2_t701800872_s30272_p2', 'uid': 0, 'uname': 'root', 'mtime': 1257141531, 'rmajor': 0, 'machine': ('Linux', 'stkendca18a.fnal.gov', '2.6.9-89.0.9.ELsmp', '#1 SMP Mon Aug 24 13:47:43 CDT 2009', 'i686'), 'sanity_size': 65536, 'type': 'cpio_odc', 'pstat': (16877, 820663368L, 20L, 1, 1602, 2752, 512L, 1257130346, 1257130346, 1254398469), 'mode_octal': '0100016', 'mode': 33188, 'size_bytes': 196173824L, 'gname': 'root', 'gid': 0, 'fullname': '/diskb/write-pool-2/data/003000000000000006199FC8', 'inode': 806985672L, 'minor': 20}, 'version': 'v3_8b CVS $Revision: 1.922 $ ', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/diskb/write-pool-2/data/003000000000000006199FC8', 'status': ('ok', None), 'outfile': '/pnfs/fnal.gov/usr/exp-db/monthly/cdf-offline/cdfofpr2/2009/11-November/01/channel01/.(access)(003000000000000006199FC8)', 'fc': {'uid': 0, 'external_label': 'VON217', 'gid': 0, 'location_cookie': '0000_000000000_0000325', 'complete_crc': 1690258267L, 'bfid': 'CDMS125714171100000', 'sanity_cookie': (65536L, 274209460L), 'size': 196173824L}, 'infilepath': '/diskb/write-pool-2/data/003000000000000006199FC8', 'at_the_top': 38, 'local_inode': 569740957L, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.13.6', 43906), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'encp_start_time': 1257141528.8155689, 'seek_time': 0.029751062393188477, 'transfer_time': 4.1740949153900146, 't0': 1257141528, 'mount_time': 0.018054008483886719, 'lm_dequeued': 1257141707.180501, 'job_queued': 1257141531.5062211, 'drive_call_time': 0.68542933464050293, 'in_queue': 175.67400884628296, 'drive_transfer_time': 3.2330710887908936}, 'user_level': 'user2', 'unique_id': 'stkendca18a.fnal.gov-12(trunc. 4098) Thread tape_thread 00:01:51 stkenmvr208a.fnal.gov 006111 root I LTO4_108MV Done is sent Thread tape_thread 00:01:51 stkenmvr208a.fnal.gov 006111 root I LTO4_108MV maybe_clean Thread tape_thread 00:01:51 stkenmvr208a.fnal.gov 006111 root I LTO4_108MV returned from maybe_clean Thread tape_thread 00:01:51 stkendca18a.fnal.gov 016282 root I ENCP Received final dialog for stkendca18a.fnal.gov-1257141531-16282-0. elapsed=182.670sec 00:01:51 stkendca18a.fnal.gov 016282 root I ENCP leaving transfer_file(). 00:01:51 stkendca18a.fnal.gov 016282 root I ENCP pf_status = ('ok', None) skip_layer_cleanup = False 00:01:51 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:01:51 stkensrv0n.fnal.gov 028796 enstore I FILSRV Updating update file set deleted = 'n', pnfs_path = '/pnfs/fnal.gov/usr/exp-db/monthly/cdf-offline/cdfofpr2/2009/11-November/01/channel01/CDFOFPR2_t701800872_s30272_p2', drive = 'stkenmvr208a:/dev/rmt/tps0d0n:1310154353', pnfs_id = '003000000000000006199FC8' where bfid = 'CDMS125714171100000'; 00:01:51 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:01:51 stkendca18a.fnal.gov 016282 root I ENCP MSG_TYPE=ENCP_XFER write_to_hsm /diskb/write-pool-2/data/003000000000000006199FC8 -> /pnfs/fnal.gov/usr/exp-db/monthly/cdf-offline/cdfofpr2/2009/11-November/01/channel01/CDFOFPR2_t701800872_s30272_p2: 196173824 bytes copied to VON217 at 44.5 MB/S (81.5 MB/S network) (57.9 MB/S drive) (97.8 MB/S disk) mover=LTO4_108.mover drive_id=ULTRIUM-TD4 drive_sn=1310154353 drive_vendor=IBM elapsed=183.08 {'storage_group': 'exp-db', 'encp_crc': 1690258267L, 'disk_rate': 97.78523447113524, 'drive_rate': 57.86632349304962, 'driver': 'FTTDriver', 'transfer_rate': 44.538576615296492, 'mover_interface': 'stkenmvr208a', 'overall_rate': 40.094865828551036, 'media_changer': 'SL8500G1.media_changer', 'network_rate': 44.538576615296492, 'encp_ip': '131.225.13.6', 'unique_id': 'stkendca18a.fnal.gov-1257141531-16282-0'} 00:01:51 stkendca18a.fnal.gov 016282 root I ENCP File /pnfs/fnal.gov/usr/exp-db/monthly/cdf-offline/cdfofpr2/2009/11-November/01/channel01/CDFOFPR2_t701800872_s30272_p2 copied successfully. 00:01:51 stkendca18a.fnal.gov 016282 root I ENCP Completed transferring 196173824 bytes in 1 files in 183.076877117 sec. Overall rate = 40.1 MB/sec. Transfer rate = 44.5 MB/sec. Network rate = 81.5 MB/sec. Drive rate = 57.9 MB/sec. Disk rate = 97.8 MB/sec. Exit status = 0. 00:01:52 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:01:52 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM write_to_hsm next work on vol=VON217 mover=LTO4_108.mover requester:root 00:01:52 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM HAVE_BOUND:sending write_to_hsm /pnfs/fnal.gov/usr/exp-db/monthly/cdf-offline/cdfofpr2/2009/11-November/01/channel01/CDFOFPR2_t701800988_s30274_p3 to mover LTO4_108.mover ('131.225.13.204', 7868) DEL_DISM None 00:01:52 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM MSG_TYPE=ADD_TO_LMQ request added to work at movers queue for storage group : exp-db 00:01:52 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM HAVE_BOUND:sending {'vc': {'storage_group': 'exp-db', 'external_label': 'VON217', 'library': 'CD-LTO4G1', 'file_family': 'monthly-cdf-offline', 'wrapper': 'cpio_odc', 'address': ('131.225.13.59', 7502), 'volume_family': 'exp-db.monthly-cdf-offline.cpio_odc', 'file_family_width': 2}, 'outfilepath': '/pnfs/fnal.gov/usr/exp-db/monthly/cdf-offline/cdfofpr2/2009/11-November/01/channel01/CDFOFPR2_t701800988_s30274_p3', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 1, 'delayed_dismount': None, 'curpri': 1}, 'file_size': 2147475456L, 'mover': 'LTO4_108.mover', 'ignore_fair_share': None, 'retry': None, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fnal.gov/usr/exp-db/monthly/cdf-offline/cdfofpr2/2009/11-November/01/channel01/CDFOFPR2_t701800988_s30274_p3', 'uid': 0, 'uname': 'root', 'type': 'cpio_odc', 'gid': 0, 'mtime': 1257141531, 'rmajor': 0, 'machine': ('Linux', 'stkendca18a.fnal.gov', '2.6.9-89.0.9.ELsmp', '#1 SMP Mon Aug 24 13:47:43 CDT 2009', 'i686'), 'sanity_size': 65536, 'pstat': (16877, 820663368L, 20L, 1, 1602, 2752, 512L, 1257130346, 1257130346, 1254398469), 'mode_octal': '0100016', 'mode': 33188, 'gname': 'root', 'size_bytes': 2147475456L, 'fullname': '/diskb/write-pool-2/data/00300000000000000619A108', 'inode': 806985992L, 'minor': 20}, 'version': 'v3_8b CVS $Revision: 1.922 $ ', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/diskb/write-pool-2/data/00300000000000000619A108', 'status': ('ok', None), 'outfile': '/pnfs/fnal.gov/usr/exp-db/monthly/cdf-offline/cdfofpr2/2009/11-November/01/channel01/.(access)(00300000000000000619A108)', 'fc': {'size': 2147475456L, 'external_label': 'VON217', 'address': ('131.225.13.59', 7501)}, 'infilepath': '/diskb/write-pool-2/data/00300000000000000619A108', 'at_the_top': 29, 'local_inode': 536875126L, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.13.6', 43907), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'lm_dequeued': 1257141712.1898811, 'encp_start_time': 1257141528.8126371, 'job_queued': 1257141531.522543, 't0': 1257141528, 'in_queue': 180.66706013679504}, 'user_level': 'user2', 'unique_id': 'stkendca18a.fnal.gov-1257141531-16281-0'} to mover 00:01:52 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:01:52 stkenmvr208a.fnal.gov 006111 root I LTO4_108MV WRITE_TO_HSM Thread MainThread 00:01:52 stkenmvr208a.fnal.gov 006111 root I LTO4_108MV SENDING {'vc': {'storage_group': 'exp-db', 'external_label': 'VON217', 'library': 'CD-LTO4G1', 'file_family': 'monthly-cdf-offline', 'wrapper': 'cpio_odc', 'address': ('131.225.13.59', 7502), 'volume_family': 'exp-db.monthly-cdf-offline.cpio_odc', 'file_family_width': 2}, 'outfilepath': '/pnfs/fnal.gov/usr/exp-db/monthly/cdf-offline/cdfofpr2/2009/11-November/01/channel01/CDFOFPR2_t701800988_s30274_p3', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 1, 'delayed_dismount': None, 'curpri': 1}, 'file_size': 2147475456L, 'mover': {'max_buffer': 2621440000L, 'check_written_file': 600, 'max_failures': 5, 'library': 'CD-LTO4G1.library_manager', 'local_mover': 0, 'single_filemark': 1, 'max_consecutive_failures': 3, 'norestart': 'INQ', 'port': 7868, 'mc_device': '0,5,1,11', 'compression': 0, 'serial_num': '1310154353', 'update_interval_in_bound': 2, 'update_interval': 5, 'syslog_entry': 'sense|st[0-9]', 'status': ('ok', None), 'max_rate': 73400320.0, 'vendor_id': 'IBM', 'driver': 'FTTDriver', 'host': 'stkenmvr208a', 'max_time_in_state': 1200, 'device': 'stkenmvr208a:/dev/rmt/tps0d0n', 'name': 'LTO4_108.mover', 'send_stats': 1, 'product_id': 'ULTRIUM-TD4', 'statistics_path': '/tmp/enstore/enstore/LTO4_108.stat', 'mover_address': ('131.225.13.204', 7868), 'mount_delay': 15, 'check_first_written_file': 0, 'callback_addr': ('131.225.13.204', 57962), 'data_ip': 'stkenmvr208a', 'logname': 'LTO4_108MV', 'hostip': '131.225.13.204', 'do_cleaning': 'No', 'media_changer': 'SL8500G1.media_changer'}, 'ignore_fair_share': None, 'retry': None, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fnal.gov/usr/exp-db/monthly/cdf-offline/cdfofpr2/2009/11-November/01/channel01/CDFOFPR2_t701800988_s30274_p3', 'uid': 0, 'uname': 'root', 'mtime': 1257141531, 'rmajor': 0, 'machine': ('Linux', 'stkendca18a.fnal.gov', '2.6.9-89.0.9.ELsmp', '#1 SMP Mon Aug 24 13:47:43 CDT 2009', 'i686'), 'sanity_size': 65536, 'type': 'cpio_odc', 'pstat': (16877, 820663368L, 20L, 1, 1602, 2752, 512L, 1257130346, 1257130346, 1254398469), 'mode_octal': '0100016', 'mode': 33188, 'size_bytes': 2147475456L, 'gname': 'root', 'gid': 0, 'fullname': '/diskb/write-pool-2/data/00300000000000000619A108', 'inode': 806985992L, 'minor': 20}, 'version': 'v3_8b CVS $Revision: 1.922 $ ', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/diskb/write-pool-2/data/00300000000000000619A108', 'status': ('ok', None), 'outfile': '/pnfs/fnal.gov/usr/exp-db/monthly/cdf-offline/cdfofpr2/2009/11-November/01/channel01/.(access)(00300000000000000619A108)', 'fc': {'size': 2147475456L, 'external_label': 'VON217', 'address': ('131.225.13.59', 7501)}, 'infilepath': '/diskb/write-pool-2/data/00300000000000000619A108', 'at_the_top': 29, 'local_inode': 536875126L, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.13.6', 43907), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'lm_dequeued': 1257141712.1898811, 'encp_start_time': 1257141528.8126371, 'job_queued': 1257141531.522543, 't0': 1257141528, 'in_queue': 180.66706013679504}, 'user_level': 'user2', 'unique_id': 'stkendca18a.fnal.gov-1257141531-16281-0'} Thread client_connect_thread 00:01:52 stkendca18a.fnal.gov 016281 root I ENCP Control socket ('131.225.13.6', 43907) is connected to ('131.225.13.204', 57963) for stkendca18a.fnal.gov-1257141531-16281-0. elapsed=183.440sec 00:01:52 stkendca18a.fnal.gov 016281 root I ENCP Received callback ticket from mover LTO4_108.mover for transfer stkendca18a.fnal.gov-1257141531-16281-0. elapsed=183.440sec 00:01:52 stkendca18a.fnal.gov 016281 root I ENCP Atempting to connect data socket to mover at ('131.225.13.204', 57962). 00:01:52 stkendca18a.fnal.gov 016281 root I ENCP Connecting data socket to mover (('131.225.13.204', 57962)) with interface 131.225.13.6. elapsed=183.440sec 00:01:52 stkendca18a.fnal.gov 016281 root I ENCP Data socket ('131.225.13.6', 43918) is connected to ('131.225.13.204', 57962). elapsed=183.440sec 00:01:52 stkendca18a.fnal.gov 016281 root I ENCP Opening local file. 00:01:52 stkenmvr208a.fnal.gov 006111 root I LTO4_108MV tape position: target 326 current 326 0 Thread seek_thread 00:01:52 stkendca18a.fnal.gov 016281 root I ENCP pf_status = ('ok', None) skip_layer_cleanup = False 00:01:52 stkenmvr208a.fnal.gov 006111 root I LTO4_108MV write_tape starting, bytes_to_write=2147475968 Thread tape_thread 00:01:52 stkenmvr208a.fnal.gov 006111 root I LTO4_108MV Write starting. Tape VON217 absolute location in blocks 4956726 Thread tape_thread 00:01:52 stkenmvr201a.fnal.gov 018563 root I LTO4_101MV Tape is ejected Thread media_thread 00:01:52 stkenmvr201a.fnal.gov 018563 root I LTO4_101MV dismounting VOR413 Thread media_thread 00:01:52 stkensrv4n.fnal.gov 003689 enstore I SL8500G1MC MC Overflow: 4 dismount VOR413 0,6,1,7 00:01:52 stkenmvr201a.fnal.gov 018563 root E LTO4_101MV unloadvol ('MEDIA_CHANGER_QUEUE_FULL', 0, 'retry later') Thread media_thread 00:01:52 stkenmvr201a.fnal.gov 018563 root I LTO4_101MV Media Changer returned MEDIA_CHANGER_QUEUE_FULL Thread media_thread 00:01:52 stkensrv4n.fnal.gov 003689 enstore I SL8500G1MC MC Overflow: 4 mount VOP202 0,5,1,12 00:01:52 stkenmvr151a.fnal.gov 013228 root E LTO4_51MV loadvol ('MEDIA_CHANGER_QUEUE_FULL', 0, 'retry later') Thread media_thread 00:01:52 stkenmvr151a.fnal.gov 013228 root I LTO4_51MV Media Changer returned MEDIA_CHANGER_QUEUE_FULL Thread media_thread 00:01:52 stkenmvr217a.fnal.gov 007075 root I LTO4_117MV Tape is ejected Thread media_thread 00:01:52 stkenmvr217a.fnal.gov 007075 root I LTO4_117MV dismounting VOO268 Thread media_thread 00:01:53 stkensrv4n.fnal.gov 005843 enstore I SL8500F1MC PROCESSING dismount VOO268 1,2,1,8 00:01:53 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:01:53 cmsstor82.fnal.gov 029584 root I ENCP File /storage/data2/write-pool-2/data/0007000000000000618C8FF8 transfered for cmsstor82.fnal.gov-1257141687-29584-0 in 17.9100141525 seconds. 00:01:53 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:01:53 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:01:54 stkenmvr146a.fnal.gov 005990 root I LTO4_46MV written bytes 2102708736/2102708736, blocks 16043 header 292 trailer 575 Thread tape_thread 00:01:55 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:01:55 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:01:55 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:01:55 stkenmvr146a.fnal.gov 005990 root I LTO4_46MV filemarks written. Tape VOR200 absolute location in blocks 4782585 Thread tape_thread 00:01:55 stkenmvr146a.fnal.gov 005990 root I LTO4_46MV write_tape timing:pull 0.20131111145 write 7.80668950081 crc_check 8.35593891144 freespace 0.217772245407 block_write 16.5817117691 idle 0.33719587326 Thread tape_thread 00:01:55 stkenmvr146a.fnal.gov 005990 root I LTO4_46MV drive stats after write. Tape VOR200 position 200 block 0 block_size 0 bloc_loc 4782585 tot_blocks 762939000 BOT 0 read_err 0 write_err 0 bytes 2102708736 block_write_tot 16.5817117691 tape_rate 126808906.419 Thread tape_thread 00:01:55 stkenmvr146a.fnal.gov 005990 root I LTO4_46MV remaining bytes info in DB 196481024000 estimated 194378315264 reported from drive 194534400000 position 201 Thread tape_thread 00:01:55 stkenmvr146a.fnal.gov 005990 root I LTO4_46MV new bitfile request {'uid': 0, 'external_label': 'VOR200', 'gid': 0, 'location_cookie': '0000_000000000_0000200', 'complete_crc': 4036583096L, 'sanity_cookie': (65536L, 1712850523L), 'size': 2102707869L} Thread tape_thread 00:01:55 stkensrv4n.fnal.gov 003689 enstore I SL8500G1MC MC Overflow: 4 dismount VON705 0,7,1,2 00:01:55 stkenmvr196a.fnal.gov 027637 root E LTO4_96MV unloadvol ('MEDIA_CHANGER_QUEUE_FULL', 0, 'retry later') Thread media_thread 00:01:55 stkenmvr196a.fnal.gov 027637 root I LTO4_96MV Media Changer returned MEDIA_CHANGER_QUEUE_FULL Thread media_thread 00:01:55 stkensrv4n.fnal.gov 003689 enstore I SL8500G1MC MC Overflow: 4 mount VOR422 0,6,1,8 00:01:55 stkenmvr206a.fnal.gov 026766 root E LTO4_106MV loadvol ('MEDIA_CHANGER_QUEUE_FULL', 0, 'retry later') Thread media_thread 00:01:55 stkenmvr206a.fnal.gov 026766 root I LTO4_106MV Media Changer returned MEDIA_CHANGER_QUEUE_FULL Thread media_thread 00:01:56 stkenmvr146a.fnal.gov 005990 root I LTO4_46MV set remaining: VOR200 194534400000 0000_000000000_0000201 Thread tape_thread 00:01:56 last message repeated 1 times 00:01:56 stkensrv0n.fnal.gov 028762 enstore I VOLSRV Updating update volume set last_access = '2009-11-02 00:01:56', eod_cookie = '0000_000000000_0000201', non_del_files = '200', remaining_bytes = 194534400000 where label = 'VOR200'; 00:01:56 stkenmvr146a.fnal.gov 005990 root I LTO4_46MV transfer complete volume=VOR200 location=201 Thread tape_thread 00:01:56 last message repeated 1 times 00:01:56 stkensrv0n.fnal.gov 028762 enstore I VOLSRV Updating update volume set sum_wr_access = '200', modification_time = '2009-11-02 00:01:56' where label = 'VOR200'; 00:01:56 stkenmvr146a.fnal.gov 005990 root I LTO4_46MV Sending done to client: {'vc': {'comment': '', 'storage_group': 'cms', 'blocksize': 131072, 'declared': 1255358745.0, 'library': 'CD-LTO4G1', 'si_time': [0.0, 1238581927.0], 'user_inhibit': ['none', 'none'], 'system_inhibit': ['none', 'none'], 'external_label': 'VOR200', 'wrapper': 'cpio_odc', 'remaining_bytes': 196481024000L, 'sum_mounts': 91, 'sum_rd_access': 0, 'file_family_width': 15, 'last_access': 1257141695.0, 'media_type': 'LTO4', 'status': ('ok', None), 'eod_cookie': '0000_000000000_0000200', 'capacity_bytes': 858993459200L, 'non_del_files': 199, 'sum_wr_err': 0, 'sum_wr_access': 199, 'file_family': 'BACKFILL2', 'address': ('131.225.13.59', 7502), 'volume_family': 'cms.BACKFILL2.cpio_odc', 'modification_time': 1257141695.0, 'write_protected': 'n', 'sum_rd_err': 0, 'first_access': 1257103365.0}, 'outfilepath': '/pnfs/fnal.gov/usr/cms/WAX//11/store/backfill/2/data/IansMagicMushroomSoup-T0Test-AnalyzeThisAndGetAFreeBigFatHairyPhD/JetMonitor/RAW/v10_31_pre20replaythingy_v15/000/000/015/4652C74E-41C6-DE11-90E3-000423DD2F34.root', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 25, 'delayed_dismount': None, 'curpri': 25}, 'file_size': 2102707869L, 'mover': {'max_buffer': 2621440000L, 'check_written_file': 600, 'max_failures': 5, 'library': 'CD-LTO4G1.library_manager', 'local_mover': 0, 'dismount_delay': 30, 'max_consecutive_failures': 3, 'norestart': 'INQ', 'port': 7846, 'mc_device': '0,6,1,12', 'compression': 0, 'serial_num': '9311032672', 'update_interval_in_bound': 2, 'update_interval': 5, 'syslog_entry': 'sense|st[0-9]', 'status': ('ok', None), 'max_rate': 73400320.0, 'vendor_id': 'IBM', 'driver': 'FTTDriver', 'host': 'stkenmvr146a', 'max_time_in_state': 1200, 'device': 'stkenmvr146a:/dev/rmt/tps0d0n', 'name': 'LTO4_46.mover', 'send_stats': 1, 'product_id': 'ULTRIUM-TD4', 'statistics_path': '/tmp/enstore/enstore/LTO4_46.stat', 'single_filemark': 1, 'mount_delay': 15, 'mover_address': ('131.225.13.174', 7846), 'check_first_written_file': 0, 'callback_addr': ('131.225.13.174', 56683), 'data_ip': 'stkenmvr146a', 'logname': 'LTO4_46MV', 'hostip': '131.225.13.174', 'do_cleaning': 'No', 'media_changer': 'SL8500G1.media_changer', 'min_buffer': 67108864L}, 'ignore_fair_share': None, 'retry': None, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fnal.gov/usr/cms/WAX//11/store/backfill/2/data/IansMagicMushroomSoup-T0Test-AnalyzeThisAndGetAFreeBigFatHairyPhD/JetMonitor/RAW/v10_31_pre20replaythingy_v15/000/000/015/4652C74E-41C6-DE11-90E3-000423DD2F34.root', 'uid': 0, 'uname': 'root', 'mtime': 1257141687, 'rmajor': 0, 'machine': ('Linux', 'cmsstor82.fnal.gov', '2.6.31', '#1 SMP Mon Sep 14 09:43:20 CDT 2009', 'x86_64'), 'sanity_size': 65536, 'type': 'cpio_odc', 'pstat': (16877, 126372832, 21L, 1, 9811, 5063, 512, 1257141604, 1257141604, 1257107163), 'mode_octal': '0100016', 'mode': 33188, 'size_bytes': 2102707869L, 'gname': 'root', 'gid': 0, 'fullname': '/storage/data2/write-pool-2/data/0007000000000000618C8FF8', 'inode': 126652408L, 'minor': 21}, 'version': 'v3_7d CVS $Revision: 1.922 $ encp', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/storage/data2/write-pool-2/data/0007000000000000618C8FF8', 'status': ('ok', None), 'outfile': '/pnfs/fnal.gov/usr/cms/WAX//11/store/backfill/2/data/IansMagicMushroomSoup-T0Test-AnalyzeThisAndGetAFreeBigFatHairyPhD/JetMonitor/RAW/v10_31_pre20replaythingy_v15/000/000/015/.(access)(0007000000000000618C8FF8)', 'fc': {'uid': 0, 'external_label': 'VOR200', 'gid': 0, 'location_cookie': '0000_000000000_0000200', 'complete_crc': 4036583096L, 'bfid': 'CDMS125714171500000', 'sanity_cookie': (65536L, 1712850523L), 'size': 2102707869L}, 'infilepath': '/storage/data2/write-pool-2/data/0007000000000000618C8FF8', 'at_the_top': 27, 'local_inode': 2147484221L, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.189.82', 41450), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'encp_start_time': 1257141686.3958831, 'seek_time': 0.034456014633178711(trunc. 4428) Thread tape_thread 00:01:56 stkenmvr146a.fnal.gov 005990 root I LTO4_46MV Done is sent Thread tape_thread 00:01:56 stkenmvr146a.fnal.gov 005990 root I LTO4_46MV maybe_clean Thread tape_thread 00:01:56 stkenmvr146a.fnal.gov 005990 root I LTO4_46MV returned from maybe_clean Thread tape_thread 00:01:56 cmsstor82.fnal.gov 029584 root I ENCP Received final dialog for cmsstor82.fnal.gov-1257141687-29584-0. elapsed=29.720sec 00:01:56 cmsstor82.fnal.gov 029584 root I ENCP leaving transfer_file(). 00:01:56 cmsstor82.fnal.gov 029584 root I ENCP pf_status = ('ok', None) skip_layer_cleanup = False 00:01:56 stkensrv0n.fnal.gov 028796 enstore I FILSRV Updating update file set deleted = 'n', pnfs_path = '/pnfs/fnal.gov/usr/cms/WAX//11/store/backfill/2/data/IansMagicMushroomSoup-T0Test-AnalyzeThisAndGetAFreeBigFatHairyPhD/JetMonitor/RAW/v10_31_pre20replaythingy_v15/000/000/015/4652C74E-41C6-DE11-90E3-000423DD2F34.root', drive = 'stkenmvr146a:/dev/rmt/tps0d0n:9311032672', pnfs_id = '0007000000000000618C8FF8' where bfid = 'CDMS125714171500000'; 00:01:56 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:01:56 cmsstor82.fnal.gov 029584 root I ENCP MSG_TYPE=ENCP_XFER write_to_hsm /storage/data2/write-pool-2/data/0007000000000000618C8FF8 -> /pnfs/fnal.gov/usr/cms/WAX//11/store/backfill/2/data/IansMagicMushroomSoup-T0Test-AnalyzeThisAndGetAFreeBigFatHairyPhD/JetMonitor/RAW/v10_31_pre20replaythingy_v15/000/000/015/4652C74E-41C6-DE11-90E3-000423DD2F34.root: 2102707869 bytes copied to VOR200 at 98.2 MB/S (89.3 MB/S network) (108 MB/S drive) (112 MB/S disk) mover=LTO4_46.mover drive_id=ULTRIUM-TD4 drive_sn=9311032672 drive_vendor=IBM elapsed=30.147 {'storage_group': 'cms', 'encp_crc': 4036583096L, 'disk_rate': 111.99528824744021, 'drive_rate': 107.7282411970296, 'driver': 'FTTDriver', 'transfer_rate': 98.156301292829937, 'mover_interface': 'stkenmvr146a', 'overall_rate': 95.720009572420352, 'media_changer': 'SL8500G1.media_changer', 'network_rate': 98.156301292829937, 'encp_ip': '131.225.189.82', 'unique_id': 'cmsstor82.fnal.gov-1257141687-29584-0'} 00:01:56 cmsstor82.fnal.gov 029584 root I ENCP File /pnfs/fnal.gov/usr/cms/WAX//11/store/backfill/2/data/IansMagicMushroomSoup-T0Test-AnalyzeThisAndGetAFreeBigFatHairyPhD/JetMonitor/RAW/v10_31_pre20replaythingy_v15/000/000/015/4652C74E-41C6-DE11-90E3-000423DD2F34.root copied successfully. 00:01:56 cmsstor82.fnal.gov 029584 root I ENCP Completed transferring 2102707869 bytes in 1 files in 30.1480460167 sec. Overall rate = 95.7 MB/sec. Transfer rate = 98.2 MB/sec. Network rate = 89.3 MB/sec. Drive rate = 108 MB/sec. Disk rate = 112 MB/sec. Exit status = 0. 00:01:57 stkenmvr218a.fnal.gov 008054 root I LTO4_118MV Tape is ejected Thread media_thread 00:01:57 stkenmvr218a.fnal.gov 008054 root I LTO4_118MV dismounting VOO437 Thread media_thread 00:01:57 stkensrv4n.fnal.gov 005843 enstore I SL8500F1MC PROCESSING dismount VOO437 1,2,1,11 00:01:57 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:01:57 stkensrv4n.fnal.gov 003689 enstore I SL8500G1MC MC Overflow: 4 dismount VOR201 0,7,1,1 00:01:57 stkenmvr198a.fnal.gov 015662 root E LTO4_98MV unloadvol ('MEDIA_CHANGER_QUEUE_FULL', 0, 'retry later') Thread media_thread 00:01:57 stkenmvr198a.fnal.gov 015662 root I LTO4_98MV Media Changer returned MEDIA_CHANGER_QUEUE_FULL Thread media_thread 00:01:57 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:01:58 cmsstor73.fnal.gov 007334 root I ENCP Version: v3_7d CVS $Revision: 1.922 $ encp OS: Linux 2.6.31 x86_64 Release: Scientific Linux SL release 5.2 (Boron) Rocks release 5.1 (V.I) User: root(0) Group: root(0) Euser: root(0) Egroup: root(0) Library: CD-LTO4G1 Storage Group: cms File Family: BACKFILL2 FF Wrapper: cpio_odc FF Width: 15 Current working directory: cmsstor73.fnal.gov:/ Command line: /home/enstore/enstore/bin/encp --verbose 4 --threaded --ecrc --resubmit-timeout 1800 --age-time 30 --delpri 1 --priority 25 --pnfs-mount /pnfs/fs --shortcut --override-path /pnfs/fnal.gov/usr/cms/WAX//11/store/backfill/2/data/IansMagicMushroomSoup-T0Test-AnalyzeThisAndGetAFreeBigFatHairyPhD/JetMonitor/RAW/v10_31_pre20replaythingy_v15/000/000/015/F818CD29-3DC6-DE11-AFC1-000423D94990.root --put-cache 0007000000000000618C8FD0 /storage/data1/write-pool-1/data/0007000000000000618C8FD0 00:01:58 cmsstor73.fnal.gov 007334 root I ENCP Creating zero length output files. 00:01:58 cmsstor73.fnal.gov 007334 root I ENCP Sending /pnfs/fnal.gov/usr/cms/WAX//11/store/backfill/2/data/IansMagicMushroomSoup-T0Test-AnalyzeThisAndGetAFreeBigFatHairyPhD/JetMonitor/RAW/v10_31_pre20replaythingy_v15/000/000/015/F818CD29-3DC6-DE11-AFC1-000423D94990.root write request to LM: unique_id: cmsstor73.fnal.gov-1257141718-7334-0 inputfile: /storage/data1/write-pool-1/data/0007000000000000618C8FD0 outputfile: /pnfs/fnal.gov/usr/cms/WAX//11/store/backfill/2/data/IansMagicMushroomSoup-T0Test-AnalyzeThisAndGetAFreeBigFatHairyPhD/JetMonitor/RAW/v10_31_pre20replaythingy_v15/000/000/015/.(access)(0007000000000000618C8FD0) 00:01:58 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM MSG_TYPE=ADD_TO_LMQ request added to pending queue for storage group : cms 00:01:58 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM write Q'd /storage/data1/write-pool-1/data/0007000000000000618C8FD0 (cmsstor73.fnal.gov-1257141718-7334-0) -> /pnfs/fnal.gov/usr/cms/WAX//11/store/backfill/2/data/IansMagicMushroomSoup-T0Test-AnalyzeThisAndGetAFreeBigFatHairyPhD/JetMonitor/RAW/v10_31_pre20replaythingy_v15/000/000/015/F818CD29-3DC6-DE11-AFC1-000423D94990.root : library=CD-LTO4G1 family=BACKFILL2 requester:root volume_family:cms.BACKFILL2.cpio_odc 00:01:58 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:01:58 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM write_to_hsm next work on vol=VOR200 mover=LTO4_46.mover requester:root 00:01:58 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM HAVE_BOUND:sending write_to_hsm /pnfs/fnal.gov/usr/cms/WAX//11/store/backfill/2/data/IansMagicMushroomSoup-T0Test-AnalyzeThisAndGetAFreeBigFatHairyPhD/JetMonitor/RAW/v10_31_pre20replaythingy_v15/000/000/015/F818CD29-3DC6-DE11-AFC1-000423D94990.root to mover LTO4_46.mover ('131.225.13.174', 7846) DEL_DISM None 00:01:58 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM MSG_TYPE=ADD_TO_LMQ request added to work at movers queue for storage group : cms 00:01:58 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM HAVE_BOUND:sending {'vc': {'storage_group': 'cms', 'external_label': 'VOR200', 'library': 'CD-LTO4G1', 'file_family': 'BACKFILL2', 'wrapper': 'cpio_odc', 'address': ('131.225.13.59', 7502), 'volume_family': 'cms.BACKFILL2.cpio_odc', 'file_family_width': 15}, 'outfilepath': '/pnfs/fnal.gov/usr/cms/WAX//11/store/backfill/2/data/IansMagicMushroomSoup-T0Test-AnalyzeThisAndGetAFreeBigFatHairyPhD/JetMonitor/RAW/v10_31_pre20replaythingy_v15/000/000/015/F818CD29-3DC6-DE11-AFC1-000423D94990.root', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 25, 'delayed_dismount': None, 'curpri': 25}, 'file_size': 2104994383L, 'mover': 'LTO4_46.mover', 'ignore_fair_share': None, 'retry': None, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fnal.gov/usr/cms/WAX//11/store/backfill/2/data/IansMagicMushroomSoup-T0Test-AnalyzeThisAndGetAFreeBigFatHairyPhD/JetMonitor/RAW/v10_31_pre20replaythingy_v15/000/000/015/F818CD29-3DC6-DE11-AFC1-000423D94990.root', 'uid': 0, 'uname': 'root', 'type': 'cpio_odc', 'gid': 0, 'mtime': 1257141718, 'rmajor': 0, 'machine': ('Linux', 'cmsstor73.fnal.gov', '2.6.31', '#1 SMP Mon Sep 14 09:43:20 CDT 2009', 'x86_64'), 'sanity_size': 65536, 'pstat': (16877, 126372832, 21L, 1, 9811, 5063, 512, 1257141604, 1257141604, 1257107163), 'mode_octal': '0100016', 'mode': 33188, 'gname': 'root', 'size_bytes': 2104994383L, 'fullname': '/storage/data1/write-pool-1/data/0007000000000000618C8FD0', 'inode': 126652368L, 'minor': 21}, 'version': 'v3_7d CVS $Revision: 1.922 $ encp', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/storage/data1/write-pool-1/data/0007000000000000618C8FD0', 'status': ('ok', None), 'outfile': '/pnfs/fnal.gov/usr/cms/WAX//11/store/backfill/2/data/IansMagicMushroomSoup-T0Test-AnalyzeThisAndGetAFreeBigFatHairyPhD/JetMonitor/RAW/v10_31_pre20replaythingy_v15/000/000/015/.(access)(0007000000000000618C8FD0)', 'fc': {'size': 2104994383L, 'external_label': 'VOR200', 'address': ('131.225.13.59', 7501)}, 'infilepath': '/storage/data1/write-pool-1/data/0007000000000000618C8FD0', 'at_the_top': 1, 'local_inode': 1073742103, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.204.249', 38088), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'lm_dequeued': 1257141718.484129, 'encp_start_time': 1257141717.4455619, 'job_queued': 1257141718.4382169, 't0': 1257141717, 'in_queue': 0.045853137969970703}, 'user_level': 'admin', 'unique_id': 'cmsstor73.fnal.gov-1257141718-7334-0'} to mover 00:01:58 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:01:58 cmsstor73.fnal.gov 007334 root I ENCP Listening for control socket at: ('131.225.204.249', 38088) 00:01:58 cmsstor73.fnal.gov 007334 root I ENCP Waiting for mover to connect control socket at address: ('131.225.204.249', 38088) elapsed=1.050sec 00:01:58 stkenmvr146a.fnal.gov 005990 root I LTO4_46MV WRITE_TO_HSM Thread MainThread 00:01:58 stkenmvr146a.fnal.gov 005990 root I LTO4_46MV SENDING {'vc': {'storage_group': 'cms', 'external_label': 'VOR200', 'library': 'CD-LTO4G1', 'file_family': 'BACKFILL2', 'wrapper': 'cpio_odc', 'address': ('131.225.13.59', 7502), 'volume_family': 'cms.BACKFILL2.cpio_odc', 'file_family_width': 15}, 'outfilepath': '/pnfs/fnal.gov/usr/cms/WAX//11/store/backfill/2/data/IansMagicMushroomSoup-T0Test-AnalyzeThisAndGetAFreeBigFatHairyPhD/JetMonitor/RAW/v10_31_pre20replaythingy_v15/000/000/015/F818CD29-3DC6-DE11-AFC1-000423D94990.root', 'encp': {'agetime': 30, 'adminpri': -1, 'delpri': 1, 'basepri': 25, 'delayed_dismount': None, 'curpri': 25}, 'file_size': 2104994383L, 'mover': {'max_buffer': 2621440000L, 'check_written_file': 600, 'max_failures': 5, 'library': 'CD-LTO4G1.library_manager', 'local_mover': 0, 'dismount_delay': 30, 'max_consecutive_failures': 3, 'norestart': 'INQ', 'port': 7846, 'mc_device': '0,6,1,12', 'compression': 0, 'serial_num': '9311032672', 'update_interval_in_bound': 2, 'update_interval': 5, 'syslog_entry': 'sense|st[0-9]', 'status': ('ok', None), 'max_rate': 73400320.0, 'vendor_id': 'IBM', 'driver': 'FTTDriver', 'host': 'stkenmvr146a', 'max_time_in_state': 1200, 'device': 'stkenmvr146a:/dev/rmt/tps0d0n', 'name': 'LTO4_46.mover', 'send_stats': 1, 'product_id': 'ULTRIUM-TD4', 'statistics_path': '/tmp/enstore/enstore/LTO4_46.stat', 'single_filemark': 1, 'mount_delay': 15, 'mover_address': ('131.225.13.174', 7846), 'check_first_written_file': 0, 'callback_addr': ('131.225.13.174', 56686), 'data_ip': 'stkenmvr146a', 'logname': 'LTO4_46MV', 'hostip': '131.225.13.174', 'do_cleaning': 'No', 'media_changer': 'SL8500G1.media_changer', 'min_buffer': 67108864L}, 'ignore_fair_share': None, 'retry': None, 'lm': {'address': ('131.225.13.4', 8541)}, 'wrapper': {'major': 0, 'rminor': 0, 'pnfsFilename': '/pnfs/fnal.gov/usr/cms/WAX//11/store/backfill/2/data/IansMagicMushroomSoup-T0Test-AnalyzeThisAndGetAFreeBigFatHairyPhD/JetMonitor/RAW/v10_31_pre20replaythingy_v15/000/000/015/F818CD29-3DC6-DE11-AFC1-000423D94990.root', 'uid': 0, 'uname': 'root', 'mtime': 1257141718, 'rmajor': 0, 'machine': ('Linux', 'cmsstor73.fnal.gov', '2.6.31', '#1 SMP Mon Sep 14 09:43:20 CDT 2009', 'x86_64'), 'sanity_size': 65536, 'type': 'cpio_odc', 'pstat': (16877, 126372832, 21L, 1, 9811, 5063, 512, 1257141604, 1257141604, 1257107163), 'mode_octal': '0100016', 'mode': 33188, 'size_bytes': 2104994383L, 'gname': 'root', 'gid': 0, 'fullname': '/storage/data1/write-pool-1/data/0007000000000000618C8FD0', 'inode': 126652368L, 'minor': 21}, 'version': 'v3_7d CVS $Revision: 1.922 $ encp', 'encp_daq': None, 'client_crc': 1, 'crc_seed': 0, 'infile': '/storage/data1/write-pool-1/data/0007000000000000618C8FD0', 'status': ('ok', None), 'outfile': '/pnfs/fnal.gov/usr/cms/WAX//11/store/backfill/2/data/IansMagicMushroomSoup-T0Test-AnalyzeThisAndGetAFreeBigFatHairyPhD/JetMonitor/RAW/v10_31_pre20replaythingy_v15/000/000/015/.(access)(0007000000000000618C8FD0)', 'fc': {'size': 2104994383L, 'external_label': 'VOR200', 'address': ('131.225.13.59', 7501)}, 'infilepath': '/storage/data1/write-pool-1/data/0007000000000000618C8FD0', 'at_the_top': 1, 'local_inode': 1073742103, 'resend': {'max_retry': 3, 'resubmit': 0, 'max_resubmits': None, 'retry': 0}, 'callback_addr': ('131.225.204.249', 38088), 'work': 'write_to_hsm', 'override_ro_mount': 0, 'times': {'lm_dequeued': 1257141718.484129, 'encp_start_time': 1257141717.4455619, 'job_queued': 1257141718.4382169, 't0': 1257141717, 'in_queue': 0.045853137969970703}, 'user_level': 'admin', 'unique_id': 'cmsstor73.fnal.gov-1257141718-7334-0'} Thread client_connect_thread 00:01:58 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:01:58 cmsstor73.fnal.gov 007334 root I ENCP Control socket ('131.225.204.249', 38088) is connected to ('131.225.13.174', 56687) for cmsstor73.fnal.gov-1257141718-7334-0. elapsed=1.100sec 00:01:58 cmsstor73.fnal.gov 007334 root I ENCP Received callback ticket from mover LTO4_46.mover for transfer cmsstor73.fnal.gov-1257141718-7334-0. elapsed=1.110sec 00:01:58 cmsstor73.fnal.gov 007334 root I ENCP Atempting to connect data socket to mover at ('131.225.13.174', 56686). 00:01:58 cmsstor73.fnal.gov 007334 root I ENCP Connecting data socket to mover (('131.225.13.174', 56686)) with interface 131.225.204.249. elapsed=1.110sec 00:01:58 cmsstor73.fnal.gov 007334 root I ENCP Data socket ('131.225.204.249', 54775) is connected to ('131.225.13.174', 56686). elapsed=1.110sec 00:01:58 cmsstor73.fnal.gov 007334 root I ENCP Opening local file. 00:01:58 cmsstor73.fnal.gov 007334 root I ENCP pf_status = ('ok', None) skip_layer_cleanup = False 00:01:58 stkenmvr146a.fnal.gov 005990 root I LTO4_46MV tape position: target 201 current 201 0 Thread seek_thread 00:01:58 stkenmvr146a.fnal.gov 005990 root I LTO4_46MV write_tape starting, bytes_to_write=2104994816 Thread tape_thread 00:01:58 stkenmvr146a.fnal.gov 005990 root I LTO4_46MV Write starting. Tape VOR200 absolute location in blocks 4782585 Thread tape_thread 00:01:58 stkensrv4n.fnal.gov 030269 enstore I SL8500G1MC mount VOR198 0,5,1,7 => 0,Mount: VOR198 mounted on 0, 5, 1, 7 00:01:58 stkensrv4n.fnal.gov 003689 enstore I SL8500G1MC FINISHED mount VOR198 0,5,1,7 returned ('ok', 0, 'mount VOR198 0,5,1,7 => 0,Mount: VOR198 mounted on 0, 5, 1, 7', '', '') 00:01:58 last message repeated 1 times 00:01:58 stkensrv0n.fnal.gov 028762 enstore I VOLSRV Updating update volume set sum_mounts = '85', last_access = '2009-11-02 00:01:58' where label = 'VOR198'; 00:01:58 stkenmvr209a.fnal.gov 006118 root I LTO4_109MV MSG_TYPE=MC_LOAD_DONE mounted VOR198 ULTRIUM-TD4 00:01:58 Thread media_thread 00:01:58 stkendca22a.fnal.gov 023987 root I ENCP Version: v3_8b CVS $Revision: 1.922 $ OS: Linux 2.6.9-89.0.9.ELsmp i686 Release: Scientific Linux Fermi LTS release 4.4 (Wilson) User: root(0) Group: root(0) Euser: root(0) Egroup: root(0) Library: garbage Storage Group: test File Family: dcache FF Wrapper: cpio_odc FF Width: 1 Current working directory: stkendca22a.fnal.gov:/home/enstore/dcache-log Command line: /fnal/ups/prd/encp/v3_8b/Linux-2-6-dcache/encp --verbose 4 --threaded --ecrc --bypass-filesystem-max-filesize-check --pnfs-mount /pnfs/fs --shortcut --override-path /pnfs/fnal.gov/usr/fermigrid/volatile/fermilab/timur-test/broken-lib/timur-test-0003 --put-cache 0005000000000000005FA5F8 /diskb/write-pool-2/data/0005000000000000005FA5F8 00:01:59 stkendca22a.fnal.gov 024070 root I ENCP Version: v3_8b CVS $Revision: 1.922 $ OS: Linux 2.6.9-89.0.9.ELsmp i686 Release: Scientific Linux Fermi LTS release 4.4 (Wilson) User: root(0) Group: root(0) Euser: root(0) Egroup: root(0) Library: garbage Storage Group: test File Family: dcache FF Wrapper: cpio_odc FF Width: 1 Current working directory: stkendca22a.fnal.gov:/home/enstore/dcache-log Command line: /fnal/ups/prd/encp/v3_8b/Linux-2-6-dcache/encp --verbose 4 --threaded --ecrc --bypass-filesystem-max-filesize-check --pnfs-mount /pnfs/fs --shortcut --override-path /pnfs/fnal.gov/usr/fermigrid/volatile/fermilab/timur-test/broken-lib/timur-test-0002 --put-cache 0005000000000000005FA5C8 /diskb/write-pool-2/data/0005000000000000005FA5C8 00:01:59 stkendca22a.fnal.gov 024088 root I ENCP Version: v3_8b CVS $Revision: 1.922 $ OS: Linux 2.6.9-89.0.9.ELsmp i686 Release: Scientific Linux Fermi LTS release 4.4 (Wilson) User: root(0) Group: root(0) Euser: root(0) Egroup: root(0) Library: garbage Storage Group: test File Family: dcache FF Wrapper: cpio_odc FF Width: 1 Current working directory: stkendca22a.fnal.gov:/home/enstore/dcache-log Command line: /fnal/ups/prd/encp/v3_8b/Linux-2-6-dcache/encp --verbose 4 --threaded --ecrc --bypass-filesystem-max-filesize-check --pnfs-mount /pnfs/fs --shortcut --override-path /pnfs/fnal.gov/usr/fermigrid/volatile/fermilab/timur-test/broken-lib/timur-test-0001 --put-cache 0005000000000000005FA590 /diskb/write-pool-2/data/0005000000000000005FA590 00:01:59 stkendca22a.fnal.gov 023987 root I ENCP Configuration Server: no such name: 'garbage.library_manager' 00:01:59 stkendca22a.fnal.gov 023987 root E ENCP INFILE=/diskb/write-pool-2/data/0005000000000000005FA5F8 OUTFILE=/pnfs/fnal.gov/usr/fermigrid/volatile/fermilab/timur-test/broken-lib/timur-test-0003 FILESIZE=801512 LABEL= LOCATION= DRIVE= DRIVE_SN= TRANSFER_TIME=0.00 SEEK_TIME=0.00 MOUNT_TIME=0.00 QWAIT_TIME=0.00 TIME2NOW=0.90 CRC= STATUS=KEYERROR Configuration Server: no such name: 'garbage.library_manager' 00:01:59 stkendca22a.fnal.gov 024070 root I ENCP Configuration Server: no such name: 'garbage.library_manager' 00:01:59 stkendca22a.fnal.gov 024070 root E ENCP INFILE=/diskb/write-pool-2/data/0005000000000000005FA5C8 OUTFILE=/pnfs/fnal.gov/usr/fermigrid/volatile/fermilab/timur-test/broken-lib/timur-test-0002 FILESIZE=801512 LABEL= LOCATION= DRIVE= DRIVE_SN= TRANSFER_TIME=0.00 SEEK_TIME=0.00 MOUNT_TIME=0.00 QWAIT_TIME=0.00 TIME2NOW=0.91 CRC= STATUS=KEYERROR Configuration Server: no such name: 'garbage.library_manager' 00:01:59 stkendca22a.fnal.gov 024088 root I ENCP Configuration Server: no such name: 'garbage.library_manager' 00:01:59 stkendca22a.fnal.gov 024088 root E ENCP INFILE=/diskb/write-pool-2/data/0005000000000000005FA590 OUTFILE=/pnfs/fnal.gov/usr/fermigrid/volatile/fermilab/timur-test/broken-lib/timur-test-0001 FILESIZE=801512 LABEL= LOCATION= DRIVE= DRIVE_SN= TRANSFER_TIME=0.00 SEEK_TIME=0.00 MOUNT_TIME=0.00 QWAIT_TIME=0.00 TIME2NOW=0.93 CRC= STATUS=KEYERROR Configuration Server: no such name: 'garbage.library_manager' 00:01:59 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM 00:01:59 stkensrv4n.fnal.gov 005357 enstore I LTO4G1LM