root filesystem - lost space

From: Alan Douglas (alan.douglas_at_mentec.ie)
Date: 12/13/04

  • Next message: Alan Douglas: "SUMMARY: root filesystem - lost space" 
    Date: Mon, 13 Dec 2004 15:17:11 +0000
To: tru64-unix-managers@ornl.gov

    Hi All,

    I have a problem similar to ones seen before, but I can't get to the
    bottom of it.
    I have a root file system of 2.7GB and it thinks its running at 95%, it
    was 100% but I moved some files.

    I have run the following (below), but I can't find the space anywhere.
    On suggestions from previous posts, I looked for lost files, but I'm not
    sure how to proceed. The system is stopping oracle sessions, it was
    rebooted this morning.

    OS is v5.1a PK4. I can't even patch it with so little space available to
    me.
    VAR is on a separate partition, so its not log related.
    I'm not sure about the fuser results, or how much space this relates to.

    All help appreciated.

    # df -k
    Filesystem 1024-blocks Used Available Capacity Mounted
    on
    root_domain#root 2788720 2620014 156216 95% /
    /proc 0 0 0 100% /proc
    usr_domain#usr 5540008 951635 4571840 18% /usr
    var_domain#var 5633792 90512 5534856 2% /var

    # du -x -k
    8 ./.tags
    8 ./cluster/members/member0/dev/pts
    8 ./cluster/members/member0/dev/streams/xtiso
    16 ./cluster/members/member0/dev/streams
    8 ./cluster/members/member0/dev/sad
    8 ./cluster/members/member0/dev/lat
    8 ./cluster/members/member0/dev/.audit
    96 ./cluster/members/member0/dev
    10 ./cluster/members/member0/sbin/it.d/data
    8 ./cluster/members/member0/sbin/it.d/23.d
    26 ./cluster/members/member0/sbin/it.d
    15 ./cluster/members/member0/sbin/init.d
    49 ./cluster/members/member0/sbin
    8 ./cluster/members/member0/devices/none
    16 ./cluster/members/member0/devices
    8 ./cluster/members/member0/etc/ppp
    8 ./cluster/members/member0/etc/atm
    118 ./cluster/members/member0/etc
    8 ./cluster/members/member0/opt
    8 ./cluster/members/member0/tmp/.oracle
    8 ./cluster/members/member0/tmp/.X11-unix
    24 ./cluster/members/member0/tmp
    8 ./cluster/members/member0/.osonly_tmp
    327 ./cluster/members/member0
    335 ./cluster/members
    46 ./cluster/admin/run
    54 ./cluster/admin
    8 ./cluster/auth/keys
    16 ./cluster/auth
    413 ./cluster
    14 ./etc/sec/file_objects
    94 ./etc/sec
    15 ./etc/sia
    60 ./etc/zoneinfo/Africa
    12 ./etc/zoneinfo/America/Indiana
    10 ./etc/zoneinfo/America/Kentucky
    144 ./etc/zoneinfo/America
    15 ./etc/zoneinfo/Antarctica
    9 ./etc/zoneinfo/Arctic
    79 ./etc/zoneinfo/Asia
    21 ./etc/zoneinfo/Atlantic
    18 ./etc/zoneinfo/Australia
    8 ./etc/zoneinfo/Brazil
    8 ./etc/zoneinfo/Canada
    9 ./etc/zoneinfo/Chile
    38 ./etc/zoneinfo/Etc
    56 ./etc/zoneinfo/Europe
    19 ./etc/zoneinfo/Indian
    8 ./etc/zoneinfo/Mexico
    43 ./etc/zoneinfo/Pacific
    27 ./etc/zoneinfo/SystemV
    8 ./etc/zoneinfo/US
    315 ./etc/zoneinfo/sources
    912 ./etc/zoneinfo
    153 ./etc/join
    24 ./etc/ppp.common
    40 ./etc/uucp
    19 ./etc/namedb/bin
    10 ./etc/namedb/src
    40 ./etc/namedb
    8 ./etc/vol/type/fsgen/fs.d
    16 ./etc/vol/type/fsgen
    24 ./etc/vol/type
    32 ./etc/vol
    54 ./etc/hwmgr
    11 ./etc/auth/subsystems
    147 ./etc/auth/system
    166 ./etc/auth
    8 ./etc/fdmns/root_domain
    8 ./etc/fdmns/usr_domain
    8 ./etc/fdmns/var_domain
    8 ./etc/fdmns/u01_domain
    8 ./etc/fdmns/u02_domain
    8 ./etc/fdmns/u03_domain
    56 ./etc/fdmns
    65 ./etc/class
    8 ./etc/ssh2/knownhosts
    8 ./etc/ssh2/hostkeys
    12 ./etc/ssh2/subconfig
    62 ./etc/ssh2
    3292 ./etc
    8 ./mnt
    8 ./opt
    18 ./sbin/it.d/bin/dnconvdir
    110 ./sbin/it.d/bin
    118 ./sbin/it.d
    12 ./sbin/init.d/bin
    283 ./sbin/init.d
    8 ./sbin/rc0.d
    8 ./sbin/rc2.d
    8 ./sbin/rc3.d
    2019 ./sbin/advfs
    38 ./sbin/lsm.d/bin
    10 ./sbin/lsm.d/fsgen/fs.d/advfs
    10 ./sbin/lsm.d/fsgen/fs.d/ufs
    28 ./sbin/lsm.d/fsgen/fs.d
    204 ./sbin/lsm.d/fsgen
    541 ./sbin/lsm.d/gen
    549 ./sbin/lsm.d/raid5
    1340 ./sbin/lsm.d
    14 ./sbin/i2o
    23569 ./sbin
    8 ./home
    6685 ./shlib
    8 ./devices/cport
    8 ./devices/disk
    8 ./devices/rdisk
    8 ./devices/tape
    2500448 ./devices/ntape
    8 ./devices/changer
    8 ./devices/dmapi
    2500504 ./devices
    44 ./mdec
    49 ./subsys
    47233 ./sys/BINARY
    47241 ./sys
    8 ./.dt/types/fp_dynamic
    24 ./.dt/types
    8 ./.dt/icons
    8 ./.dt/appmanager
    26 ./.dt/help/root-ds25-0
    34 ./.dt/help
    14 ./.dt/sessions/current
    14 ./.dt/sessions/current.old
    37 ./.dt/sessions
    8 ./.dt/tmp
    8 ./.dt/Trash
    8 ./.dt/Desktop
    146 ./.dt
    12 ./tcb/bin
    56 ./tcb/files
    41 ./tcb/lib
    117 ./tcb
    8 ./lost+found
    8 ./.sysman
    59 ./TT_DB
    8 ./cdrom
    110 ./backup
    30 ./mentec/SYSCHECK130.kit/instctrl
    3790 ./mentec/SYSCHECK130.kit
    31 ./mentec/SYSCHECK133.kit/instctrl
    3831 ./mentec/SYSCHECK133.kit
    7680 ./mentec
    24 ./.netscape/cache
    8 ./.netscape/archive
    9 ./.netscape/xover-cache/host-news
    17 ./.netscape/xover-cache
    340 ./.netscape
    8 ./nsmail
    10 ./.ssh2/hostkeys
    19 ./.ssh2
    2619966 .
    # fuser -d /

    fuser: 2 process(es) referencing deleted files
           PID FD INODE TYPE BLK_CNT REFERENCE

           593 3 2 socket 0 open
           593 4 5950 regular 0 open
           593 5 0 socket 0 open
           593 6 1 socket 0 open
           650 5 244 fifo 0 open
           650 7 6076 regular 0 open

    # ps -ef | grep 593
    root 593 1 0.0 10:04:39 ?? 0:00.00
    /usr/sbin/svrMgt_mib
    root 17717 16698 0.0 14:58:48 pts/3 0:00.00 grep 593

    # ps -ef | grep 650
    root 650 1 0.0 10:04:40 ?? 0:02.36
    /usr/sbin/envmond -ui cli
    root 653 650 0.0 - ?? 0:00.00 <defunct>
    root 20807 16698 0.0 15:07:15 pts/3 0:00.00 grep 650

    # ls -laF
    total 30898
    drwxr-xr-x 31 root system 8192 Dec 13 14:48 ./
    drwxr-xr-x 31 root system 8192 Dec 13 14:48 ../
    -rw------- 1 root system 220 Dec 13 10:06 .TTauthority
    -rw------- 1 root system 198 Dec 13 12:27 .Xauthority
    -rwxr----- 1 root system 1322 Aug 1 2001 .cshrc*
    drwxr-xr-x 10 root system 8192 Dec 13 10:06 .dt/
    -rwxr-xr-x 1 root system 3970 May 21 2003 .dtprofile*
    lrwxr-xr-x 1 root system 22 May 21 2003 .local..@ ->
    cluster/members/{memb}/
    -rwxr----- 1 root system 1990 Aug 1 2001 .login*
    -rwxr--r-- 1 root system 2433 Aug 1 2001 .mrg...login*
    drwx------ 5 root system 8192 Jan 7 2004 .netscape/
    -rwxr----- 1 root system 1322 Aug 1 2001 .new...cshrc*
    -rwxr----- 1 root system 1990 Aug 1 2001 .new...login*
    -rwxr----- 1 root system 1864 Aug 1 2001 .new...profile*
    lrwxrwxrwx 1 root system 34 May 21 2003 .osonly_tmp@ ->
    cluster/members/{memb}/.osonly_tmp/
    -rwxr----- 1 root system 1864 Aug 1 2001 .profile*
    -rwxr----- 1 root system 1322 Aug 1 2001 .proto...cshrc*
    -rwxr----- 1 root system 1990 Aug 1 2001 .proto...login*
    -rwxr----- 1 root system 1864 Aug 1 2001 .proto...profile*
    drwxr-xr-x 3 root system 8192 Jan 16 2004 .ssh2/
    drwxr-xr-x 2 root system 8192 May 21 2003 .sysman/
    drwx------ 2 root system 8192 May 21 2003 .tags/
    -rw-r--r-- 1 root system 0 May 21 2003 DS25
    -rw-r--r-- 1 root system 0 May 21 2003 DS25.devs
    -rw-r--r-- 1 bin bin 2203 May 21 2003 GENERIC
    drwxr-xr-x 2 root system 8192 May 21 2003 TT_DB/
    drwxr-xr-x 2 root system 8192 Jun 5 2003 backup/
    lrwxr-xr-x 1 root system 7 May 21 2003 bin@ -> usr/bin/
    drwxr-xr-x 2 root system 8192 May 21 2003 cdrom/
    -rwxr-xr-x 1 root system 176 Nov 26 17:15 checkshutdown.sh*
    -rwxr-xr-x 1 root system 32 Nov 26 16:10 cleanshutdown.sh*
    drwxr-xr-x 5 root system 8192 May 22 2003 cluster/
    lrwxrwxrwx 1 root system 26 May 21 2003 dev@ ->
    cluster/members/{memb}/dev/
    drwxr-xr-x 9 root system 8192 May 21 2003 devices/
    -rwxr-xr-x 1 root system 32 Nov 23 16:29 downnow.sh*
    drwxr-xr-x 15 root system 16384 Dec 13 12:27 etc/
    -rwxr-xr-x 1 root system 12013104 Jan 24 2003 genvmunix*
    drwxr-xr-x 2 root system 8192 May 21 2003 home/
    -rw-r--r-- 1 root system 10442 May 21 2003 java.txt
    lrwxr-xr-x 1 root system 7 May 21 2003 lib@ -> usr/lib/
    dr-x------ 2 root system 8192 Dec 13 14:48 lost+found/
    drwxr-xr-x 2 root system 8192 May 21 2003 mdec/
    drwxr-xr-x 2 root system 8192 May 21 2003 mnt/
    -rw------- 1 root system 0 May 21 2003 nohup.out
    drwx------ 2 root system 8192 Dec 11 2003 nsmail/
    drwxr-xr-x 2 root system 8192 May 21 2003 opt/
    -rwxr-xr-x 1 root system 104656 Jan 24 2003 osf_boot*
    dr-xr-xr-x 2 root system 1048608 Dec 13 15:12 proc/
    -rw-r----- 1 root operator 16384 May 22 2003 quota.group
    -rw-r----- 1 root operator 16384 May 21 2003 quota.user
    drwxr-xr-x 10 root system 8192 May 21 2003 sbin/
    drwxr-xr-x 2 root system 8192 May 21 2003 shlib/
    drwxr-xr-x 2 root system 8192 Aug 1 2001 subsys/
    drwxr-xr-x 3 root system 8192 May 21 2003 sys/
    -rw-r--r-- 1 root system 5281 Nov 21 2003 t.t
    drwxr-xr-x 5 root system 8192 Aug 1 2001 tcb/
    lrwxrwxrwx 1 root system 26 May 21 2003 tmp@ ->
    cluster/members/{memb}/tmp/
    drwxr-xr-x 27 root system 8192 May 21 2003 usr/
    drwxr-xr-x 23 root system 8192 Feb 6 2004 var/
    -rwxr-xr-x 1 root system 18123856 May 21 2003 vmunix*

    CONFIDENTIALITY NOTICE This email is private and confidential and may contain legally privileged information.If you are not named above as an addressee it may be unlawful for you to read, copy, distribute, disclose or otherwise use the information in this email. If you are not the intended recipient of this email please contact our systems manager. Thank you. Any views or opinions presented are solely those of the author and do not necessarily represent those of the company or its associated companies unless otherwise specifically stated.

  • Next message: Alan Douglas: "SUMMARY: root filesystem - lost space"